From 0f8773bde442e7edca0bf8c699dd2855d5b56db4 Mon Sep 17 00:00:00 2001 From: John Cardinal Date: Mon, 11 Oct 2021 19:56:14 +0000 Subject: [PATCH] --- server/AyaNova/biz/Search.cs | 184 +++++++++++++++++++++++------------ 1 file changed, 123 insertions(+), 61 deletions(-) diff --git a/server/AyaNova/biz/Search.cs b/server/AyaNova/biz/Search.cs index 1f9636fc..9f5ff96b 100644 --- a/server/AyaNova/biz/Search.cs +++ b/server/AyaNova/biz/Search.cs @@ -107,86 +107,148 @@ namespace AyaNova.Biz -/* -this query will do what I want properly. The wildcards here are just for wildcard terms and can also be direct matches doesn't matter the end result is only matches where all terms match the record -however they are desired i.e. AND not OR which is better than v7 which was shit at this as the wildcard search returned all it's results and so did any other terms -So I just need to build this query in this form from the search query terms, put them into the MatchingObjects collection and then the rest will fall into place and can resume as normal + /* + this query will do what I want properly. The wildcards here are just for wildcard terms and can also be direct matches doesn't matter the end result is only matches where all terms match the record + however they are desired i.e. AND not OR which is better than v7 which was shit at this as the wildcard search returned all it's results and so did any other terms + So I just need to build this query in this form from the search query terms, put them into the MatchingObjects collection and then the rest will fall into place and can resume as normal -WITH qr AS (SELECT aSearchKey.ObjectID, aSearchKey.aType, -count(*) FILTER (WHERE asearchdictionary.word like '%1qt%') AS "st1", -count(*) FILTER (WHERE asearchdictionary.word like '%44%') AS "st2" -FROM aSearchDictionary INNER JOIN aSearchKey ON aSearchDictionary.id = aSearchKey.WordID -group by asearchkey.objectid, asearchkey.atype) -select objectid, atype FROM qr -WHERE st1 > 0 and st2 >0 -order by atype, objectid + WITH qr AS (SELECT aSearchKey.ObjectID, aSearchKey.aType, + count(*) FILTER (WHERE asearchdictionary.word like '%1qt%') AS "st1", + count(*) FILTER (WHERE asearchdictionary.word like '%44%') AS "st2" + FROM aSearchDictionary INNER JOIN aSearchKey ON aSearchDictionary.id = aSearchKey.WordID + group by asearchkey.objectid, asearchkey.atype) + select objectid, atype FROM qr + WHERE st1 > 0 and st2 >0 + order by atype, objectid -*/ + */ - - - //List holder for matching dictionary ID's - List DictionaryMatches = new List(); + //============================== START SEARCH ========================================================== + #region old search + // //List holder for matching dictionary ID's + // List DictionaryMatches = new List(); - //GET LIST OF DICTIONARY ID'S THAT MATCH REGULAR SEARCH TERMS - if (SearchTerms.Count > 0) - foreach (string Term in SearchTerms) - { - DictionaryMatches.AddRange(await ct.SearchDictionary.Where(z => z.Word==Term).Select(z => z.Id).ToListAsync()); - } + // //GET LIST OF DICTIONARY ID'S THAT MATCH REGULAR SEARCH TERMS + // if (SearchTerms.Count > 0) + // foreach (string Term in SearchTerms) + // { + // DictionaryMatches.AddRange(await ct.SearchDictionary.Where(z => z.Word==Term).Select(z => z.Id).ToListAsync()); + // } - //GET LIST OF DICTIONARY ID'S THAT MATCH WILDCARD SEARCH TERMS - if (PreWildCardedSearchTerms.Count > 0) + // //GET LIST OF DICTIONARY ID'S THAT MATCH WILDCARD SEARCH TERMS + // if (PreWildCardedSearchTerms.Count > 0) + // { + // foreach (string WildCardSearchTerm in PreWildCardedSearchTerms) + // { + // //Contains? + // if (WildCardSearchTerm.StartsWith("%") && WildCardSearchTerm.EndsWith("%")) + // { + // DictionaryMatches.AddRange(await ct.SearchDictionary.Where(z => z.Word.Contains(WildCardSearchTerm.Replace("%", ""))).Select(z => z.Id).ToListAsync()); + // } + // else if (WildCardSearchTerm.EndsWith("%")) //STARTS WITH? + // { + // DictionaryMatches.AddRange(await ct.SearchDictionary.Where(z => z.Word.StartsWith(WildCardSearchTerm.Replace("%", ""))).Select(z => z.Id).ToListAsync()); + // } + // else if (WildCardSearchTerm.StartsWith("%"))//ENDS WITH? + // { + // DictionaryMatches.AddRange(await ct.SearchDictionary.Where(z => z.Word.EndsWith(WildCardSearchTerm.Replace("%", ""))).Select(z => z.Id).ToListAsync()); + // } + // } + // } + + // //SEARCH SEARCHKEY FOR MATCHING WORDS AND OPTIONALLY TYPE + // var TotalSearchTermsToMatch = PreWildCardedSearchTerms.Count + SearchTerms.Count; + + // //Build search query based on searchParameters + // var q = ct.SearchKey.Distinct().Where(z => DictionaryMatches.Contains(z.WordId)); + + + // //Of type? + // if (searchParameters.TypeOnly != AyaType.NoType) + // q = q.Where(z => z.AType == searchParameters.TypeOnly); + + + // //Find the records that have the search terms in searchkey + // var SearchMatches = q.GroupBy(z => new { z.AType, z.ObjectId }).Select(z => new { ObjectId = z.Key.ObjectId, AType = z.Key.AType, ObjectCount = z.LongCount() }); + + // //PUT THE RESULTS INTO MATCHING OBJECTS LIST + // foreach (var SearchMatch in SearchMatches) + // { + // //keep any object that matches *all* the search terms + // if (SearchMatch.ObjectCount >= TotalSearchTermsToMatch) + // MatchingObjects.Add(new AyaTypeId(SearchMatch.AType, SearchMatch.ObjectId)); + // } + + + #endregion old search + + + StringBuilder q = new StringBuilder(); + int termCount = 0; + + q.Append("WITH qr AS (SELECT asearchkey.atype, asearchkey.objectid, "); + + //EXACT MATCH SEARCH TERMS + foreach (string Term in SearchTerms) + q.Append($"COUNT(*) FILTER (WHERE asearchdictionary.word = '{Term}') AS 'st{++termCount}', "); + + //WILDCARD SEARCH TERMS + foreach (string WildCardSearchTerm in PreWildCardedSearchTerms) + q.Append($"COUNT(*) FILTER (WHERE asearchdictionary.word LIKE '{WildCardSearchTerm}') AS 'st{++termCount}', "); + + q.Append("FROM asearchdictionary INNER JOIN asearchkey ON asearchdictionary.id = asearchkey.wordid GROUP BY asearchkey.objectid, asearchkey.atype) SELECT objectid, atype FROM qr WHERE "); + + for (; termCount > 0; termCount--) + q.Append($"st{termCount} > 0 {(termCount > 1 ? "AND" : "")}"); + + + //execute the query and iterate the results + using (var command = ct.Database.GetDbConnection().CreateCommand()) { - foreach (string WildCardSearchTerm in PreWildCardedSearchTerms) + await ct.Database.OpenConnectionAsync(); + command.CommandText = q.ToString(); + // try + // { + using (var dr = await command.ExecuteReaderAsync()) { - //Contains? - if (WildCardSearchTerm.StartsWith("%") && WildCardSearchTerm.EndsWith("%")) + while (dr.Read()) { - DictionaryMatches.AddRange(await ct.SearchDictionary.Where(z => z.Word.Contains(WildCardSearchTerm.Replace("%", ""))).Select(z => z.Id).ToListAsync()); - } - else if (WildCardSearchTerm.EndsWith("%")) //STARTS WITH? - { - DictionaryMatches.AddRange(await ct.SearchDictionary.Where(z => z.Word.StartsWith(WildCardSearchTerm.Replace("%", ""))).Select(z => z.Id).ToListAsync()); - } - else if (WildCardSearchTerm.StartsWith("%"))//ENDS WITH? - { - DictionaryMatches.AddRange(await ct.SearchDictionary.Where(z => z.Word.EndsWith(WildCardSearchTerm.Replace("%", ""))).Select(z => z.Id).ToListAsync()); + MatchingObjects.Add(new AyaTypeId((AyaType)dr.GetInt32(0), dr.GetInt64(1))); } } + // } + // catch (Npgsql.PostgresException e) + // { + // //log out the exception and the query + // log.LogError("DataListFetcher:GetIdListResponseAsync query failed unexpectedly. IDList Query was:"); + // log.LogError(qDataQuery); + + // log.LogError(e, "DB Exception"); + // throw new System.Exception("DataListFetcher:GetIdListResponseAsync - Query failed see log"); + // } + // catch (System.Exception e) + // { + // //ensure any other type of exception gets surfaced properly + // //log out the exception and the query + // log.LogError("DataListFetcher:GetIdListResponseAsync unexpected failure. IDList Query was:"); + // log.LogError(qDataQuery); + + // log.LogError(e, "Exception"); + // throw new System.Exception("DataListFetcher:GetIdListResponseAsync - unexpected failure see log"); + // } } - - //SEARCH SEARCHKEY FOR MATCHING WORDS AND OPTIONALLY TYPE - var TotalSearchTermsToMatch = PreWildCardedSearchTerms.Count + SearchTerms.Count; - - //Build search query based on searchParameters - var q = ct.SearchKey.Distinct().Where(z => DictionaryMatches.Contains(z.WordId)); + //============================================== END SEARCH ============================================================ - //Of type? - if (searchParameters.TypeOnly != AyaType.NoType) - q = q.Where(z => z.AType == searchParameters.TypeOnly); - //Find the records that have the search terms in searchkey - var SearchMatches = q.GroupBy(z => new { z.AType, z.ObjectId }).Select(z => new { ObjectId = z.Key.ObjectId, AType = z.Key.AType, ObjectCount = z.LongCount() }); - - - //PUT THE RESULTS INTO MATCHING OBJECTS LIST - foreach (var SearchMatch in SearchMatches) - { - //keep any object that matches *all* the search terms - if (SearchMatch.ObjectCount >= TotalSearchTermsToMatch) - MatchingObjects.Add(new AyaTypeId(SearchMatch.AType, SearchMatch.ObjectId)); - } - //REMOVE ANY ITEMS THAT USER IS NOT PERMITTED TO READ //list to hold temporary matches @@ -631,7 +693,7 @@ order by atype, objectid return this; } - + public SearchIndexProcessObjectParameters AddText(long l) { Words.Add(l.ToString()); @@ -799,7 +861,7 @@ order by atype, objectid { #region Include token //All latin text is converted to lower case - c = char.ToLower(c,System.Globalization.CultureInfo.CurrentCulture); + c = char.ToLower(c, System.Globalization.CultureInfo.CurrentCulture); //Do we already have a word? if (sbWord.Length > 0) @@ -866,7 +928,7 @@ order by atype, objectid { #region Latin Include token //All latin text is converted to lower case - c = char.ToLower(c,System.Globalization.CultureInfo.CurrentCulture); + c = char.ToLower(c, System.Globalization.CultureInfo.CurrentCulture); //Do we already have a word? if (sbWord.Length > 0)