From 23c02641271c40ff83436cf2dfb82f58ba71daaa Mon Sep 17 00:00:00 2001 From: John Cardinal Date: Wed, 22 Jan 2020 20:58:28 +0000 Subject: [PATCH] --- devdocs/todo.txt | 3 + server/AyaNova/DataList/DataListFetcher.cs | 19 +++- .../DataListSqlFilterCriteriaBuilder.cs | 98 +++++++++++-------- 3 files changed, 73 insertions(+), 47 deletions(-) diff --git a/devdocs/todo.txt b/devdocs/todo.txt index 274cabdc..8d7a3b0c 100644 --- a/devdocs/todo.txt +++ b/devdocs/todo.txt @@ -5,6 +5,9 @@ eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOiIxNTcxODU5OTU0IiwiZXhwIjoiMTU3MjQ ## IMMEDIATE ITEMS +TODO: DataFilter how to distinguish between filtering on specific ID value or on value column + - Might need to add a filter on ID type of thing maybe? + - Wait and see...YAGNI scenario maybe INTEGRATION TEST UPDATE Update tests when appropriate (when it's working fully and not mocked) UPDATE SPEC DOCS with new format and names etc or at least remove nonsense as necessary diff --git a/server/AyaNova/DataList/DataListFetcher.cs b/server/AyaNova/DataList/DataListFetcher.cs index 995e489a..939f0db5 100644 --- a/server/AyaNova/DataList/DataListFetcher.cs +++ b/server/AyaNova/DataList/DataListFetcher.cs @@ -27,7 +27,7 @@ namespace AyaNova.DataList //check rights if (listOptions.Mini) - { + { if (!UserRoles.HasAnyFlags(DataList.MiniListAllowedRoles)) throw new System.UnauthorizedAccessException("User roles insufficient for this mini format datalist"); } @@ -82,7 +82,7 @@ namespace AyaNova.DataList { TheFilter = await ct.DataListFilter.FirstOrDefaultAsync(x => x.Id == listOptions.DataFilterId); //WHERE CLAUSE - FILTER - qWhere = DataListSqlFilterCriteriaBuilder.DataFilterToSQLCriteria(TheFilter, DataList.FieldDefinitions, UserId); + qWhere = DataListSqlFilterCriteriaBuilder.DataFilterToSQLCriteria(DataList.FieldDefinitions, TheFilter, DataList.FieldDefinitions, UserId); //ORDER BY CLAUSE - SORT //BUILD ORDER BY AND APPEND IT qOrderBy = DataListSqlFilterOrderByBuilder.DataFilterToSQLOrderBy(TheFilter); @@ -129,7 +129,16 @@ namespace AyaNova.DataList //PROCESS THE DF DEFAULT FIRST COLUMN //first column is always the underlying id value of the default record to open for this row in the client ui - row.Add(new AyaFieldData() { v = dr.GetInt64(0) }); + if (!dr.IsDBNull(0)) + { + row.Add(new AyaFieldData() { v = dr.GetInt64(0) }); + } + else + { +#if (DEBUG) + throw new System.ArgumentNullException($"DEV ERROR in DataListFetcher.cs: fetching df column for {DataListKey} df value is null, expecting long int record value"); +#endif + } //GetOrdinal by name is flakey in npgsql so just going by field definition and ordinal numerically int nCurrentColumnPointer = 1;//start at 1 @@ -146,7 +155,9 @@ namespace AyaNova.DataList if (f.SqlIdColumnName != null)//skip over df column id, it's not there { - AyaField.i = dr.GetInt64(nCurrentColumnPointer); + if (!dr.IsDBNull(nCurrentColumnPointer)) + AyaField.i = dr.GetInt64(nCurrentColumnPointer); + nCurrentColumnPointer++; } row.Add(AyaField); diff --git a/server/AyaNova/DataList/DataListSqlFilterCriteriaBuilder.cs b/server/AyaNova/DataList/DataListSqlFilterCriteriaBuilder.cs index e015dcd5..fe6d32e6 100644 --- a/server/AyaNova/DataList/DataListSqlFilterCriteriaBuilder.cs +++ b/server/AyaNova/DataList/DataListSqlFilterCriteriaBuilder.cs @@ -11,8 +11,9 @@ namespace AyaNova.DataList { public static class DataListSqlFilterCriteriaBuilder { - public static string DataFilterToSQLCriteria(AyaNova.Models.DataListFilter dataFilter, List objectFields, long userId) + public static string DataFilterToSQLCriteria(List objectFieldsList, AyaNova.Models.DataListFilter dataFilter, List objectFields, long userId) { + //BUGBUG: is using field property name not sql value column or whatever it's supposed to be if (string.IsNullOrWhiteSpace(dataFilter.Filter)) { @@ -40,7 +41,18 @@ namespace AyaNova.DataList var dataType = objectFields.Find(x => x.FieldKey.ToLowerInvariant() == fld).UiFieldDataType; sb.Append("("); - sb.Append(DataFilterToColumnCriteria(fld, (AyaUiFieldDataType)dataType, opType, val, tagList, userId)); + + //Get the correct sql column name + AyaDataListFieldDefinition DataListField = objectFieldsList.FirstOrDefault(x => x.FieldKey == fld); +#if (DEBUG) + //Developers little helper + if (DataListField == null) + { + throw new System.ArgumentNullException($"DEV ERROR in DataListSqlFilterCriteriaBuilder.cs: field {fld} specified in template was NOT found in ObjectFields list"); + } +#endif + var SQLValueColumnName=DataListField.GetSqlValueColumnName(); + sb.Append(DataFilterToColumnCriteria(SQLValueColumnName, (AyaUiFieldDataType)dataType, opType, val, tagList, userId)); if (i < FilterArray.Count - 1) { sb.Append(") AND "); @@ -53,13 +65,13 @@ namespace AyaNova.DataList /// /// Translate DataFilter to Postgres friendly SQL criteria /// - private static string DataFilterToColumnCriteria(string sColumn, AyaUiFieldDataType DataType, string sOperator, string sValue, List sTags, long userId)//, bool IsCompound) + private static string DataFilterToColumnCriteria(string SqlColumnNameToFilter, AyaUiFieldDataType DataType, string sOperator, string sValue, List sTags, long userId)//, bool IsCompound) { bool TagFilter = sTags.Count < 0; StringBuilder sb = new StringBuilder(); //Column name //sb.Append(" "); - sb.Append(sColumn); + sb.Append(SqlColumnNameToFilter); sb.Append(" "); //handle null values separately @@ -74,7 +86,7 @@ namespace AyaNova.DataList sb.Append("Is Null"); sb.Append(" OR "); - sb.Append(sColumn); + sb.Append(SqlColumnNameToFilter); sb.Append(" = ''"); } else @@ -121,12 +133,12 @@ namespace AyaNova.DataList break; case DataListFilterComparisonOperator.LessThan: sb.Append("Is Null OR "); - sb.Append(sColumn); + sb.Append(SqlColumnNameToFilter); sb.Append(" "); break; case DataListFilterComparisonOperator.LessThanOrEqualTo: sb.Append("Is Null OR "); - sb.Append(sColumn); + sb.Append(SqlColumnNameToFilter); sb.Append(" "); break; // case "Like": @@ -135,7 +147,7 @@ namespace AyaNova.DataList case DataListFilterComparisonOperator.NotEqual: //This is the big one: sb.Append("Is Null OR "); - sb.Append(sColumn); + sb.Append(SqlColumnNameToFilter); sb.Append(" "); break; } @@ -327,21 +339,21 @@ namespace AyaNova.DataList dtAfter = RelativeToday.AddDays(-1); dtAfter = dtAfter.AddSeconds(-1); dtBefore = RelativeToday;//.AddDays(-1); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.Today: //Between yesterday at midnight and tommorow at midnight dtAfter = RelativeToday.AddSeconds(-1); dtBefore = RelativeToday.AddDays(1); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.Tomorrow: //Between Tonight at midnight and day after tommorow at midnight dtAfter = RelativeToday.AddDays(1); dtAfter = dtAfter.AddSeconds(-1); dtBefore = RelativeToday.AddDays(2); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; //Case 402 @@ -361,7 +373,7 @@ namespace AyaNova.DataList dtAfter = dtAfter.AddSeconds(-1); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; @@ -390,7 +402,7 @@ namespace AyaNova.DataList dtBefore = dtBefore.AddDays(1); } - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.NextWeek: //Between Next Sunday at midnight and Next Next sunday at midnight @@ -413,7 +425,7 @@ namespace AyaNova.DataList //case 1155 dtAfter = dtAfter.AddSeconds(-1); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.LastMonth: //start with the first day of this month @@ -429,7 +441,7 @@ namespace AyaNova.DataList // 'yyyy-mm-ddTHH:MM:SS' - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.ThisMonth: //start with the first day of this month @@ -441,7 +453,7 @@ namespace AyaNova.DataList //case 1155 dtAfter = dtAfter.AddSeconds(-1); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.NextMonth: @@ -460,7 +472,7 @@ namespace AyaNova.DataList dtAfter = dtAfter.AddSeconds(-1); //TODO: REMOVE THIS TESTING // log.LogInformation($"NEXT MONTH TOKEN BUILDER IMMEDIATELY BEFORE BUILDING FRAGMENT: BEFORE={dtBefore.ToString()} AND AFTER={dtAfter.ToString()}"); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.FourteenDayWindow: //start with today zero hour @@ -473,7 +485,7 @@ namespace AyaNova.DataList //case 1155 dtAfter = dtAfter.AddSeconds(-1); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; @@ -483,14 +495,14 @@ namespace AyaNova.DataList //Forever up to Now dtAfter = new DateTime(1753, 1, 2, 00, 00, 00); dtBefore = DateTime.UtcNow; - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.Future: //From Now to forever (999 years from now) dtAfter = DateTime.UtcNow; dtBefore = DateTime.UtcNow.AddYears(999); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.LastYear: @@ -498,7 +510,7 @@ namespace AyaNova.DataList dtAfter = new DateTime(RelativeNow.AddYears(-1).Year, 1, 1, 0, 0, 00).AddSeconds(-1).AddHours(TimeZoneAdjustment); //To zero hour January 1 this year dtBefore = new DateTime(RelativeNow.Year, 1, 1, 0, 0, 00).AddHours(TimeZoneAdjustment); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.ThisYear: @@ -506,7 +518,7 @@ namespace AyaNova.DataList dtAfter = new DateTime(RelativeNow.Year, 1, 1, 00, 00, 00).AddSeconds(-1).AddHours(TimeZoneAdjustment); //To zero hour Jan 1 next year dtBefore = new DateTime(RelativeNow.AddYears(1).Year, 1, 1, 00, 00, 00).AddHours(TimeZoneAdjustment); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.InTheLast3Months: @@ -514,7 +526,7 @@ namespace AyaNova.DataList dtAfter = DateTime.UtcNow.AddMonths(-3); //To Now dtBefore = DateTime.UtcNow; - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.InTheLast6Months: @@ -522,7 +534,7 @@ namespace AyaNova.DataList dtAfter = DateTime.UtcNow.AddMonths(-6); //To Now dtBefore = DateTime.UtcNow; - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.InTheLastYear: @@ -530,7 +542,7 @@ namespace AyaNova.DataList dtAfter = DateTime.UtcNow.AddDays(-365); //To Now dtBefore = DateTime.UtcNow; - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; //======================= @@ -542,7 +554,7 @@ namespace AyaNova.DataList dtAfter = new DateTime(RelativeNow.Year, 1, 1, 00, 00, 00).AddSeconds(-1).AddHours(TimeZoneAdjustment); ; //To now dtBefore = RelativeNow; - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.Past90Days: @@ -550,21 +562,21 @@ namespace AyaNova.DataList dtAfter = DateTime.UtcNow.AddDays(-90); //To Now dtBefore = DateTime.UtcNow; - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.Past30Days: //From Now minus 30 days dtAfter = DateTime.UtcNow.AddDays(-30); //To Now dtBefore = DateTime.UtcNow; - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.Past24Hours: //From Now minus 24 hours dtAfter = DateTime.UtcNow.AddHours(-24); //To Now dtBefore = DateTime.UtcNow; - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.January: @@ -572,67 +584,67 @@ namespace AyaNova.DataList dtAfter = new DateTime(RelativeNow.Year, 1, 1, 00, 00, 00).AddSeconds(-1).AddHours(TimeZoneAdjustment); //To zero hour feb 1 this year dtBefore = new DateTime(RelativeNow.Year, 2, 1, 00, 00, 00).AddHours(TimeZoneAdjustment); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.February: dtAfter = new DateTime(RelativeNow.Year, 2, 1, 00, 00, 00).AddSeconds(-1).AddHours(TimeZoneAdjustment); dtBefore = new DateTime(RelativeNow.Year, 3, 1, 00, 00, 00).AddHours(TimeZoneAdjustment); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.March: dtAfter = new DateTime(RelativeNow.Year, 3, 1, 00, 00, 00).AddSeconds(-1).AddHours(TimeZoneAdjustment); dtBefore = new DateTime(RelativeNow.Year, 4, 1, 00, 00, 00).AddHours(TimeZoneAdjustment); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.April: dtAfter = new DateTime(RelativeNow.Year, 4, 1, 00, 00, 00).AddSeconds(-1).AddHours(TimeZoneAdjustment); dtBefore = new DateTime(RelativeNow.Year, 5, 1, 00, 00, 00).AddHours(TimeZoneAdjustment); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.May: dtAfter = new DateTime(RelativeNow.Year, 5, 1, 00, 00, 00).AddSeconds(-1).AddHours(TimeZoneAdjustment); dtBefore = new DateTime(RelativeNow.Year, 6, 1, 00, 00, 00).AddHours(TimeZoneAdjustment); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.June: dtAfter = new DateTime(RelativeNow.Year, 6, 1, 00, 00, 00).AddSeconds(-1).AddHours(TimeZoneAdjustment); dtBefore = new DateTime(RelativeNow.Year, 7, 1, 00, 00, 00).AddHours(TimeZoneAdjustment); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.July: dtAfter = new DateTime(RelativeNow.Year, 7, 1, 00, 00, 00).AddSeconds(-1).AddHours(TimeZoneAdjustment); dtBefore = new DateTime(RelativeNow.Year, 8, 1, 00, 00, 00).AddHours(TimeZoneAdjustment); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.August: dtAfter = new DateTime(RelativeNow.Year, 8, 1, 00, 00, 00).AddSeconds(-1).AddHours(TimeZoneAdjustment); dtBefore = new DateTime(RelativeNow.Year, 9, 1, 00, 00, 00).AddHours(TimeZoneAdjustment); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.September: dtAfter = new DateTime(RelativeNow.Year, 9, 1, 00, 00, 00).AddSeconds(-1).AddHours(TimeZoneAdjustment); dtBefore = new DateTime(RelativeNow.Year, 10, 1, 00, 00, 00).AddHours(TimeZoneAdjustment); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.October: dtAfter = new DateTime(RelativeNow.Year, 10, 1, 00, 00, 00).AddSeconds(-1).AddHours(TimeZoneAdjustment); dtBefore = new DateTime(RelativeNow.Year, 11, 1, 00, 00, 00).AddHours(TimeZoneAdjustment); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.November: dtAfter = new DateTime(RelativeNow.Year, 11, 1, 00, 00, 00).AddSeconds(-1).AddHours(TimeZoneAdjustment); dtBefore = new DateTime(RelativeNow.Year, 12, 1, 00, 00, 00).AddHours(TimeZoneAdjustment); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; case DataListFilterSpecialToken.December: @@ -640,7 +652,7 @@ namespace AyaNova.DataList dtAfter = new DateTime(RelativeNow.Year, 12, 1, 00, 00, 00).AddSeconds(-1).AddHours(TimeZoneAdjustment); //To zero hour Jan 1 next year dtBefore = new DateTime(RelativeNow.AddYears(1).Year, 1, 1, 00, 00, 00).AddHours(TimeZoneAdjustment); - BuildBetweenTwoDatesFragment(sColumn, sb, dtAfter, dtBefore); + BuildBetweenTwoDatesFragment(SqlColumnNameToFilter, sb, dtAfter, dtBefore); break; default: @@ -668,7 +680,7 @@ namespace AyaNova.DataList sb.Append(">='"); sb.Append(sLowEnd); sb.Append("' AND "); - sb.Append(sColumn); + sb.Append(SqlColumnNameToFilter); sb.Append(" "); sb.Append("<='"); sb.Append(sHighEnd); @@ -701,7 +713,7 @@ namespace AyaNova.DataList sb.Append("<'"); sb.Append(sLowEnd); sb.Append("' OR "); - sb.Append(sColumn); + sb.Append(SqlColumnNameToFilter); sb.Append(" "); sb.Append(">'"); sb.Append(sHighEnd);