From 45d705180448b15f93ab34e08e8327c3cea92400 Mon Sep 17 00:00:00 2001 From: John Cardinal Date: Fri, 7 Dec 2018 16:06:58 +0000 Subject: [PATCH] --- devdocs/specs/core-search.txt | 2 +- server/AyaNova/biz/Search.cs | 238 +++++++++++++---------------- server/AyaNova/biz/WidgetBiz.cs | 4 +- server/AyaNova/models/SearchKey.cs | 2 +- server/AyaNova/util/AySchema.cs | 6 +- 5 files changed, 113 insertions(+), 139 deletions(-) diff --git a/devdocs/specs/core-search.txt b/devdocs/specs/core-search.txt index c7019630..16f4e98d 100644 --- a/devdocs/specs/core-search.txt +++ b/devdocs/specs/core-search.txt @@ -86,7 +86,7 @@ asearchkey - objectid (long id of source object) - objecttype (AyaType as int of source object) - inname (bool indicates the search word was in the name of the object) - - intags (bool indicates teh search word was found in the tags collection for the object) + diff --git a/server/AyaNova/biz/Search.cs b/server/AyaNova/biz/Search.cs index 358d494b..a2f3d798 100644 --- a/server/AyaNova/biz/Search.cs +++ b/server/AyaNova/biz/Search.cs @@ -31,8 +31,7 @@ namespace AyaNova.Biz - Can be empty if tags are specified, no tags and no phrase is an error condition - ObjectType: only return results for objects of this type - InName: flag that indicates only search in names - - Tags that are also on result objects - - Can be empty if a phrase is specified + ACTION @@ -65,7 +64,7 @@ namespace AyaNova.Biz public string Phrase { get; set; } public bool NameOnly { get; set; } public AyaType TypeOnly { get; set; } - public List Tags { get; set; } + //Note: maxresults of 0 will get all results public int MaxResults { get; set; } @@ -73,7 +72,6 @@ namespace AyaNova.Biz { NameOnly = false; TypeOnly = AyaType.NoType; - Tags = new List(); MaxResults = 500; } @@ -84,11 +82,6 @@ namespace AyaNova.Biz //has a phrase? if (!string.IsNullOrWhiteSpace(this.Phrase)) return true; - - //has tags? - if (this.Tags.Count > 0) - return true; - return false; } } @@ -197,7 +190,6 @@ namespace AyaNova.Biz q = q.Where(m => m.ObjectType == searchParameters.TypeOnly); - //Find the records that have the search terms in searchkey var SearchMatches = q.GroupBy(x => new { x.ObjectType, x.ObjectId }).Select(x => new { ObjectId = x.Key.ObjectId, ObjectType = x.Key.ObjectType, ObjectCount = x.LongCount() }); @@ -211,102 +203,104 @@ namespace AyaNova.Biz } + #region TAGS DEPRECATED + // //--------------- TAGS ARE DEPRECATED AS BEING SOMETHING YOU CAN SEARCH FOR SEPARATELY (INTAGS) INSTEAD THEY ARE TREATED AS PART OF THE TEXT OF THE OBJECT like any other + // //IF TAGS SPECIFIED + // if (searchParameters.Tags.Count > 0) + // { + // //get a count of the search tags (used by both paths below) + // var SearchTagCount = searchParameters.Tags.Count; - //--------------- TAGS - //IF TAGS SPECIFIED - if (searchParameters.Tags.Count > 0) - { - //get a count of the search tags (used by both paths below) - var SearchTagCount = searchParameters.Tags.Count; + // if (string.IsNullOrWhiteSpace(searchParameters.Phrase)) + // { - if (string.IsNullOrWhiteSpace(searchParameters.Phrase)) - { + // #region TAGS ONLY SEARCH (NO PHRASE) ALL FULL MATCHES ARE INCLUSIVE + // Dictionary TagCounts = new Dictionary(); - #region TAGS ONLY SEARCH (NO PHRASE) ALL FULL MATCHES ARE INCLUSIVE - Dictionary TagCounts = new Dictionary(); + // //QUERY FOR ALL TAGMAPS THAT MATCH OBJECT TYPE AND ID FOR EVERY TAG SPECIFIED (UNION) + // //var tagmatches= await ct.TagMap.Where(m => ).Select(m => m.Id).ToListAsync(); + // //ct.TagMap.Where(n => n.Tags.Count(t => tags.Contains(t.DisplayName)) == tags.Count) - //QUERY FOR ALL TAGMAPS THAT MATCH OBJECT TYPE AND ID FOR EVERY TAG SPECIFIED (UNION) - //var tagmatches= await ct.TagMap.Where(m => ).Select(m => m.Id).ToListAsync(); - //ct.TagMap.Where(n => n.Tags.Count(t => tags.Contains(t.DisplayName)) == tags.Count) + // //algorithm: + // //1) get counts for each tag specified from tagmap, if any are zero then none match and can bail early + // foreach (long SearchTagId in searchParameters.Tags) + // { + // var MatchTagCount = await ct.TagMap.Where(m => m.TagId == SearchTagId).LongCountAsync(); + // //zero tags matching here at any point means no results for the entire search and we can bail + // if (MatchTagCount == 0) + // { + // //return empty resultlist + // return ReturnObject; + // } - //algorithm: - //1) get counts for each tag specified from tagmap, if any are zero then none match and can bail early - foreach (long SearchTagId in searchParameters.Tags) - { - var MatchTagCount = await ct.TagMap.Where(m => m.TagId == SearchTagId).LongCountAsync(); - //zero tags matching here at any point means no results for the entire search and we can bail - if (MatchTagCount == 0) - { - //return empty resultlist - return ReturnObject; - } + // //Save the matching count + // TagCounts.Add(SearchTagId, MatchTagCount); + // } - //Save the matching count - TagCounts.Add(SearchTagId, MatchTagCount); - } + // //2) find smallest count match so we are working with the shortest list first + // var ShortestMatchingTag = TagCounts.OrderBy(x => x.Value).First().Key; - //2) find smallest count match so we are working with the shortest list first - var ShortestMatchingTag = TagCounts.OrderBy(x => x.Value).First().Key; + // //3) Generate the shortlist of items that match the shortest tag list + // var ShortList = await ct.TagMap.Where(x => x.TagId == ShortestMatchingTag).ToListAsync(); - //3) Generate the shortlist of items that match the shortest tag list - var ShortList = await ct.TagMap.Where(x => x.TagId == ShortestMatchingTag).ToListAsync(); + // //4) Iterate the shortlist and see if each item matches all other tags specified if it does then put it into the matching objects list for return - //4) Iterate the shortlist and see if each item matches all other tags specified if it does then put it into the matching objects list for return - - //Iterate shortlist - foreach (TagMap t in ShortList) - { - var matchCount = 1; - //Iterate requested tags - foreach (long TagId in searchParameters.Tags) - { - //skipping already matched shortest tag - if (TagId != ShortestMatchingTag) - { - //Ok, does this object have this tag? - bool HasTag = await ct.TagMap.Where(x => x.TagToObjectId == t.TagToObjectId && x.TagToObjectType == t.TagToObjectType && x.TagId == TagId).AnyAsync(); - if (HasTag) - matchCount++; - } - } - //does it match all tags? - if (matchCount == SearchTagCount) - { - //yes, add it to the results - MatchingObjects.Add(new AyaTypeId(t.TagToObjectType, t.TagToObjectId)); - } - } - #endregion + // //Iterate shortlist + // foreach (TagMap t in ShortList) + // { + // var matchCount = 1; + // //Iterate requested tags + // foreach (long TagId in searchParameters.Tags) + // { + // //skipping already matched shortest tag + // if (TagId != ShortestMatchingTag) + // { + // //Ok, does this object have this tag? + // bool HasTag = await ct.TagMap.Where(x => x.TagToObjectId == t.TagToObjectId && x.TagToObjectType == t.TagToObjectType && x.TagId == TagId).AnyAsync(); + // if (HasTag) + // matchCount++; + // } + // } + // //does it match all tags? + // if (matchCount == SearchTagCount) + // { + // //yes, add it to the results + // MatchingObjects.Add(new AyaTypeId(t.TagToObjectType, t.TagToObjectId)); + // } + // } + // #endregion - } - else - { - #region TAGS PLUS PHRASE SEARCH WITH NON MATCHING TAGS EXCLUSIVE - //list to hold temporary matches - List TagMatchingObjects = new List(); + // } + // else + // { + // #region TAGS PLUS PHRASE SEARCH WITH NON MATCHING TAGS EXCLUSIVE + // //list to hold temporary matches + // List TagMatchingObjects = new List(); - //LOOP THROUGH MATCHING OBJECTS LIST - foreach (AyaTypeId i in MatchingObjects) - { - var matchCount = await ct.TagMap.Where(x => x.TagToObjectId == i.ObjectId && x.TagToObjectType == i.ObjectType && searchParameters.Tags.Contains(x.TagId)).LongCountAsync(); - if (matchCount == SearchTagCount) - { - TagMatchingObjects.Add(i); - } + // //LOOP THROUGH MATCHING OBJECTS LIST + // foreach (AyaTypeId i in MatchingObjects) + // { + // var matchCount = await ct.TagMap.Where(x => x.TagToObjectId == i.ObjectId && x.TagToObjectType == i.ObjectType && searchParameters.Tags.Contains(x.TagId)).LongCountAsync(); + // if (matchCount == SearchTagCount) + // { + // TagMatchingObjects.Add(i); + // } - } + // } - //Ok here we have all the MatchingObjects that had all the tags in the TagMatchingObjects list so that's actually now our defacto return list - MatchingObjects = TagMatchingObjects; + // //Ok here we have all the MatchingObjects that had all the tags in the TagMatchingObjects list so that's actually now our defacto return list + // MatchingObjects = TagMatchingObjects; - #endregion + // #endregion - } - } + // } + // } //--------------------- END TAGS ------------- + #endregion tags DEPRECATED + //REMOVE ANY ITEMS THAT USER IS NOT PERMITTED TO READ //If it's a name only search then all is allowed @@ -386,16 +380,13 @@ namespace AyaNova.Biz public AyaType ObjectType { get; set; } public string Name { get; set; } public List Words { get; set; } - public List Tags { get; set; } - public SearchIndexProcessObjectParameters(long localeId, long objectID, AyaType objectType, string name, List tags = null) + + public SearchIndexProcessObjectParameters(long localeId, long objectID, AyaType objectType, string name) { Words = new List(); - if (tags != null) - Tags = tags; - else - Tags = new List(); + LocaleId = localeId; ObjectId = objectID; ObjectType = objectType; @@ -413,6 +404,15 @@ namespace AyaNova.Biz Words.Add(u.ToString()); return this; } + public SearchIndexProcessObjectParameters AddWord(List lWords) + { + foreach (string s in lWords) + { + Words.Add(s); + } + + return this; + } } @@ -465,10 +465,10 @@ namespace AyaNova.Biz List NameKeyWordList = Break(p.LocaleId, p.Name); //BUILD ALL KEYWORDS LIST - List tempList = new List(); - tempList.AddRange(KeyWordList); - tempList.AddRange(p.Tags); - var DistinctAllKeywordsCombinedList = tempList.Distinct(); + // List tempList = new List(); + // tempList.AddRange(KeyWordList); + // tempList.AddRange(p.Tags); + // var DistinctAllKeywordsCombinedList = tempList.Distinct(); // AllKeyWordsCombinedList=((IEnumerable)AllKeyWordsCombinedList).Distinct(); //EARLY EXIT IF NO KEYWORDS OR NAME RECORD OR TAGS TO PROCESS @@ -482,19 +482,14 @@ namespace AyaNova.Biz List MatchingKeywordIdList = new List(); //ITERATE ALL THE KEYWORDS, SEARCH IN THE SEARCHDICTIONARY TABLE AND COLLECT ID'S OF ANY PRE-EXISTING IN DB KEYWORDS - // var ExistingKeywordMatches = ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(m => KeyWordList.Contains(m.Word)).ToDictionary(m => m.Id, m => m.Word); - var ExistingKeywordMatches = ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(m => DistinctAllKeywordsCombinedList.Contains(m.Word)).ToDictionary(m => m.Id, m => m.Word); + var ExistingKeywordMatches = ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(m => KeyWordList.Contains(m.Word)).ToDictionary(m => m.Id, m => m.Word); + //var ExistingKeywordMatches = ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(m => DistinctAllKeywordsCombinedList.Contains(m.Word)).ToDictionary(m => m.Id, m => m.Word); //Put the matching keyword ID's into the list foreach (KeyValuePair K in ExistingKeywordMatches) { //Name or regular word match? bool IsName = NameKeyWordList.Contains(K.Value); - if (IsName || KeyWordList.Contains(K.Value)) - MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = K.Key, InName = IsName }); - - //Tag match? - if (p.Tags.Contains(K.Value)) - MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = K.Key, InTags = true }); + MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = K.Key, InName = IsName }); } @@ -505,7 +500,7 @@ namespace AyaNova.Biz var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###"); #endif - foreach (string KeyWord in DistinctAllKeywordsCombinedList) + foreach (string KeyWord in KeyWordList) { if (!ExistingKeywordMatches.ContainsValue(KeyWord)) { @@ -524,16 +519,7 @@ namespace AyaNova.Biz //------- //Add to matching keywords - //was this before tags: MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = NewWord.Id, InName = NameKeyWordList.Contains(KeyWord) }); - //Name or regular word match? - bool IsName = NameKeyWordList.Contains(KeyWord); - if (IsName || KeyWordList.Contains(KeyWord)) - MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = NewWord.Id, InName = IsName }); - - //Tag match? - if (p.Tags.Contains(KeyWord)) - MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = NewWord.Id, InTags = true }); - + MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = NewWord.Id, InName = NameKeyWordList.Contains(KeyWord) }); //------- @@ -561,19 +547,7 @@ namespace AyaNova.Biz var SearchDictionaryMatchFoundInDB = ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(x => x.Word == KeyWord).FirstOrDefault(); if (SearchDictionaryMatchFoundInDB != null) { - //was before tags: MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = SearchDictionaryMatchFoundInDB.Id, InName = NameKeyWordList.Contains(KeyWord) }); - //------- - //Add to matching keywords - //Name or regular word match? - bool IsName = NameKeyWordList.Contains(KeyWord); - if (IsName || KeyWordList.Contains(KeyWord)) - MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = SearchDictionaryMatchFoundInDB.Id, InName = IsName }); - - //Tag match? - if (p.Tags.Contains(KeyWord)) - MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = SearchDictionaryMatchFoundInDB.Id, InTags = true }); - - //------- + MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = SearchDictionaryMatchFoundInDB.Id, InName = NameKeyWordList.Contains(KeyWord) }); //It exists now ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word); } @@ -605,7 +579,7 @@ namespace AyaNova.Biz var NewSearchKeyList = new List(); foreach (MatchingDictionaryEntry E in MatchingKeywordIdList) { - NewSearchKeyList.Add(new SearchKey() { WordId = E.DictionaryId, InName = E.InName, InTags = E.InTags, ObjectId = p.ObjectId, ObjectType = p.ObjectType }); + NewSearchKeyList.Add(new SearchKey() { WordId = E.DictionaryId, InName = E.InName, ObjectId = p.ObjectId, ObjectType = p.ObjectType }); } var CtSearchKeyAdd = ServiceProviderProvider.DBContext; CtSearchKeyAdd.SearchKey.AddRange(NewSearchKeyList); @@ -619,12 +593,12 @@ namespace AyaNova.Biz public class MatchingDictionaryEntry { public bool InName { get; set; } - public bool InTags { get; set; } + public long DictionaryId { get; set; } public MatchingDictionaryEntry() { InName = false; - InTags = false; + DictionaryId = -1; } } diff --git a/server/AyaNova/biz/WidgetBiz.cs b/server/AyaNova/biz/WidgetBiz.cs index 137ad725..77335ea8 100644 --- a/server/AyaNova/biz/WidgetBiz.cs +++ b/server/AyaNova/biz/WidgetBiz.cs @@ -133,8 +133,8 @@ namespace AyaNova.Biz EventLogProcessor.LogEventToDatabase(new Event(UserId, outObj.Id, BizType, AyaEvent.Created), TempContext); //SEARCH INDEXING - var SearchParams = new Search.SearchIndexProcessObjectParameters(UserLocaleId, outObj.Id, BizType, outObj.Name, outObj.Tags); - SearchParams.AddWord(outObj.Notes).AddWord(outObj.Name).AddWord(outObj.Serial); + var SearchParams = new Search.SearchIndexProcessObjectParameters(UserLocaleId, outObj.Id, BizType, outObj.Name); + SearchParams.AddWord(outObj.Notes).AddWord(outObj.Name).AddWord(outObj.Serial).AddWord(outObj.Tags); Search.ProcessNewObjectKeywords(SearchParams); return outObj; diff --git a/server/AyaNova/models/SearchKey.cs b/server/AyaNova/models/SearchKey.cs index b117e842..052da113 100644 --- a/server/AyaNova/models/SearchKey.cs +++ b/server/AyaNova/models/SearchKey.cs @@ -20,7 +20,7 @@ namespace AyaNova.Models [Required] public AyaType ObjectType { get; set; } public bool InName { get; set; } - public bool InTags { get; set; } + } } diff --git a/server/AyaNova/util/AySchema.cs b/server/AyaNova/util/AySchema.cs index b663e701..73a7b5cf 100644 --- a/server/AyaNova/util/AySchema.cs +++ b/server/AyaNova/util/AySchema.cs @@ -22,7 +22,7 @@ namespace AyaNova.Util //!!!!WARNING: BE SURE TO UPDATE THE DbUtil::PrepareDatabaseForSeeding WHEN NEW TABLES ADDED!!!! private const int DESIRED_SCHEMA_LEVEL = 8; - internal const long EXPECTED_COLUMN_COUNT = 95; + internal const long EXPECTED_COLUMN_COUNT = 94; internal const long EXPECTED_INDEX_COUNT = 20; //!!!!WARNING: BE SURE TO UPDATE THE DbUtil::PrepareDatabaseForSeeding WHEN NEW TABLES ADDED!!!! @@ -145,7 +145,7 @@ namespace AyaNova.Util //LOOKAT: this index is periodically being violated during testing exec("CREATE UNIQUE INDEX asearchdictionary_word_idx ON asearchdictionary (word);"); - exec("CREATE TABLE asearchkey (id BIGSERIAL PRIMARY KEY, wordid bigint not null REFERENCES asearchdictionary (id), objectid bigint not null, objecttype integer not null, inname bool not null, intags bool not null)"); + exec("CREATE TABLE asearchkey (id BIGSERIAL PRIMARY KEY, wordid bigint not null REFERENCES asearchdictionary (id), objectid bigint not null, objecttype integer not null, inname bool not null)"); //create locale text tables exec("CREATE TABLE alocale (id BIGSERIAL PRIMARY KEY, ownerid bigint not null, name varchar(255) not null, stock bool, cjkindex bool default false)"); @@ -299,7 +299,7 @@ namespace AyaNova.Util if (currentSchema < 8) { LogUpdateMessage(log); - + exec("CREATE TABLE adatafilter (id BIGSERIAL PRIMARY KEY, ownerid bigint not null, name varchar(255) not null, public bool not null," + "listkey varchar(255) not null, filter text, UNIQUE(name))");