From 106c00876b07aa7818a156f9a5e5a63da489ab7b Mon Sep 17 00:00:00 2001 From: John Cardinal Date: Thu, 6 Dec 2018 23:28:49 +0000 Subject: [PATCH] --- devdocs/todo.txt | 9 +- server/AyaNova/biz/Search.cs | 167 +++++++++++---------- test/raven-integration/Search/SearchOps.cs | 166 ++++++++++---------- 3 files changed, 173 insertions(+), 169 deletions(-) diff --git a/devdocs/todo.txt b/devdocs/todo.txt index e708003f..e795dd62 100644 --- a/devdocs/todo.txt +++ b/devdocs/todo.txt @@ -17,14 +17,15 @@ SON OF TAGS - Remove old tag code, models, schema tables, cleanup and db erase code that refers to them - THINGS TO FIX AS A RESULT OF THIS: - //done? - Search.cs search code had a ton of tag stuff in it, rejig that shit - - Import AyaNova7 biz tons of tag stuff + //done - Search.cs search indexing code had a ton of tag stuff in it, rejig that shit + //done - Import AyaNova7 biz tons of tag stuff - - re-introduce Tag search to search test and make sure it works properly + - re-code tag search + - re-introduce Tag search to search test and make sure it works properly with widget - Add new tag code to all taggable objects, their models and schema - Update seeder to tag objects with known set of tags for test and eval purposes - Seed widget with predictable tags (Maybe colour names "red", "green", "blue" etc) - - Test v7 import tags users with correct scheduleable user group or whatever they will be tagged with + - Test v7 import tags users with correct tags for region, sched group and dispatch zone SERVER SCHEMA - Add unique constraint to all name columns in all tables in ayschema and run tests (how did I miss that before??) diff --git a/server/AyaNova/biz/Search.cs b/server/AyaNova/biz/Search.cs index a5193771..358d494b 100644 --- a/server/AyaNova/biz/Search.cs +++ b/server/AyaNova/biz/Search.cs @@ -31,7 +31,7 @@ namespace AyaNova.Biz - Can be empty if tags are specified, no tags and no phrase is an error condition - ObjectType: only return results for objects of this type - InName: flag that indicates only search in names - - Tag ids that are also on result objects + - Tags that are also on result objects - Can be empty if a phrase is specified @@ -65,7 +65,7 @@ namespace AyaNova.Biz public string Phrase { get; set; } public bool NameOnly { get; set; } public AyaType TypeOnly { get; set; } - // public List Tags { get; set; } + public List Tags { get; set; } //Note: maxresults of 0 will get all results public int MaxResults { get; set; } @@ -73,7 +73,7 @@ namespace AyaNova.Biz { NameOnly = false; TypeOnly = AyaType.NoType; - // Tags = new List(); + Tags = new List(); MaxResults = 500; } @@ -85,9 +85,9 @@ namespace AyaNova.Biz if (!string.IsNullOrWhiteSpace(this.Phrase)) return true; - // //has tags? - // if (this.Tags.Count > 0) - // return true; + //has tags? + if (this.Tags.Count > 0) + return true; return false; } @@ -211,99 +211,102 @@ namespace AyaNova.Biz } - //IF TAGS SPECIFIED - //BUGBUG: If no valid tags provided, i.e. a single tag of type or id 0 then can skip - // if (searchParameters.Tags.Count > 0) - // { - // //get a count of the search tags (used by both paths below) - // var SearchTagCount = searchParameters.Tags.Count; - // if (string.IsNullOrWhiteSpace(searchParameters.Phrase)) - // { + //--------------- TAGS + //IF TAGS SPECIFIED + if (searchParameters.Tags.Count > 0) + { + //get a count of the search tags (used by both paths below) + var SearchTagCount = searchParameters.Tags.Count; - // #region TAGS ONLY SEARCH (NO PHRASE) ALL FULL MATCHES ARE INCLUSIVE - // Dictionary TagCounts = new Dictionary(); + if (string.IsNullOrWhiteSpace(searchParameters.Phrase)) + { - // //QUERY FOR ALL TAGMAPS THAT MATCH OBJECT TYPE AND ID FOR EVERY TAG SPECIFIED (UNION) - // //var tagmatches= await ct.TagMap.Where(m => ).Select(m => m.Id).ToListAsync(); - // //ct.TagMap.Where(n => n.Tags.Count(t => tags.Contains(t.DisplayName)) == tags.Count) + #region TAGS ONLY SEARCH (NO PHRASE) ALL FULL MATCHES ARE INCLUSIVE + Dictionary TagCounts = new Dictionary(); - // //algorithm: - // //1) get counts for each tag specified from tagmap, if any are zero then none match and can bail early - // foreach (long SearchTagId in searchParameters.Tags) - // { - // var MatchTagCount = await ct.TagMap.Where(m => m.TagId == SearchTagId).LongCountAsync(); - // //zero tags matching here at any point means no results for the entire search and we can bail - // if (MatchTagCount == 0) - // { - // //return empty resultlist - // return ReturnObject; - // } + //QUERY FOR ALL TAGMAPS THAT MATCH OBJECT TYPE AND ID FOR EVERY TAG SPECIFIED (UNION) + //var tagmatches= await ct.TagMap.Where(m => ).Select(m => m.Id).ToListAsync(); + //ct.TagMap.Where(n => n.Tags.Count(t => tags.Contains(t.DisplayName)) == tags.Count) - // //Save the matching count - // TagCounts.Add(SearchTagId, MatchTagCount); - // } + //algorithm: + //1) get counts for each tag specified from tagmap, if any are zero then none match and can bail early + foreach (long SearchTagId in searchParameters.Tags) + { + var MatchTagCount = await ct.TagMap.Where(m => m.TagId == SearchTagId).LongCountAsync(); + //zero tags matching here at any point means no results for the entire search and we can bail + if (MatchTagCount == 0) + { + //return empty resultlist + return ReturnObject; + } - // //2) find smallest count match so we are working with the shortest list first - // var ShortestMatchingTag = TagCounts.OrderBy(x => x.Value).First().Key; + //Save the matching count + TagCounts.Add(SearchTagId, MatchTagCount); + } - // //3) Generate the shortlist of items that match the shortest tag list - // var ShortList = await ct.TagMap.Where(x => x.TagId == ShortestMatchingTag).ToListAsync(); + //2) find smallest count match so we are working with the shortest list first + var ShortestMatchingTag = TagCounts.OrderBy(x => x.Value).First().Key; - // //4) Iterate the shortlist and see if each item matches all other tags specified if it does then put it into the matching objects list for return + //3) Generate the shortlist of items that match the shortest tag list + var ShortList = await ct.TagMap.Where(x => x.TagId == ShortestMatchingTag).ToListAsync(); - // //Iterate shortlist - // foreach (TagMap t in ShortList) - // { - // var matchCount = 1; - // //Iterate requested tags - // foreach (long TagId in searchParameters.Tags) - // { - // //skipping already matched shortest tag - // if (TagId != ShortestMatchingTag) - // { - // //Ok, does this object have this tag? - // bool HasTag = await ct.TagMap.Where(x => x.TagToObjectId == t.TagToObjectId && x.TagToObjectType == t.TagToObjectType && x.TagId == TagId).AnyAsync(); - // if (HasTag) - // matchCount++; - // } - // } - // //does it match all tags? - // if (matchCount == SearchTagCount) - // { - // //yes, add it to the results - // MatchingObjects.Add(new AyaTypeId(t.TagToObjectType, t.TagToObjectId)); - // } - // } - // #endregion + //4) Iterate the shortlist and see if each item matches all other tags specified if it does then put it into the matching objects list for return + + //Iterate shortlist + foreach (TagMap t in ShortList) + { + var matchCount = 1; + //Iterate requested tags + foreach (long TagId in searchParameters.Tags) + { + //skipping already matched shortest tag + if (TagId != ShortestMatchingTag) + { + //Ok, does this object have this tag? + bool HasTag = await ct.TagMap.Where(x => x.TagToObjectId == t.TagToObjectId && x.TagToObjectType == t.TagToObjectType && x.TagId == TagId).AnyAsync(); + if (HasTag) + matchCount++; + } + } + //does it match all tags? + if (matchCount == SearchTagCount) + { + //yes, add it to the results + MatchingObjects.Add(new AyaTypeId(t.TagToObjectType, t.TagToObjectId)); + } + } + #endregion - // } - // else - // { - // #region TAGS PLUS PHRASE SEARCH WITH NON MATCHING TAGS EXCLUSIVE - // //list to hold temporary matches - // List TagMatchingObjects = new List(); + } + else + { + #region TAGS PLUS PHRASE SEARCH WITH NON MATCHING TAGS EXCLUSIVE + //list to hold temporary matches + List TagMatchingObjects = new List(); - // //LOOP THROUGH MATCHING OBJECTS LIST - // foreach (AyaTypeId i in MatchingObjects) - // { - // var matchCount = await ct.TagMap.Where(x => x.TagToObjectId == i.ObjectId && x.TagToObjectType == i.ObjectType && searchParameters.Tags.Contains(x.TagId)).LongCountAsync(); - // if (matchCount == SearchTagCount) - // { - // TagMatchingObjects.Add(i); - // } + //LOOP THROUGH MATCHING OBJECTS LIST + foreach (AyaTypeId i in MatchingObjects) + { + var matchCount = await ct.TagMap.Where(x => x.TagToObjectId == i.ObjectId && x.TagToObjectType == i.ObjectType && searchParameters.Tags.Contains(x.TagId)).LongCountAsync(); + if (matchCount == SearchTagCount) + { + TagMatchingObjects.Add(i); + } - // } + } - // //Ok here we have all the MatchingObjects that had all the tags in the TagMatchingObjects list so that's actually now our defacto return list - // MatchingObjects = TagMatchingObjects; + //Ok here we have all the MatchingObjects that had all the tags in the TagMatchingObjects list so that's actually now our defacto return list + MatchingObjects = TagMatchingObjects; - // #endregion + #endregion - // } - // } + } + } + + //--------------------- END TAGS ------------- //REMOVE ANY ITEMS THAT USER IS NOT PERMITTED TO READ //If it's a name only search then all is allowed diff --git a/test/raven-integration/Search/SearchOps.cs b/test/raven-integration/Search/SearchOps.cs index 8bd084cb..70fb60c0 100644 --- a/test/raven-integration/Search/SearchOps.cs +++ b/test/raven-integration/Search/SearchOps.cs @@ -348,100 +348,100 @@ namespace raven_integration }//eot - // [Fact] - // public async void TagSearchShouldWork() - // { - // const string TEST_SEARCH_PHRASE = "element* aardvark"; + [Fact] + public async void TagSearchShouldWork() + { + const string TEST_SEARCH_PHRASE = "element* aardvark"; + + // //CREATE A TAG + // dynamic D = new JObject(); + // D.name = Util.Uniquify("TAGSEARCH"); + // ApiResponse a = await Util.PostAsync("Tag", await Util.GetTokenAsync("manager", "l3tm3in"), D.ToString()); + // Util.ValidateDataReturnResponseOk(a); + // long TagId = a.ObjectResponse["data"]["id"].Value(); + + //Tags + dynamic TagsArray = new JArray(); + TagsArray.Add(Util.Uniquify("TAGSEARCH")); + + //w.tags = InclusiveTagsArray; + + //CREATE A WIDGET + dynamic D = new JObject(); + D.name = Util.Uniquify("TAG search test WIDGET TAG AND PHRASE"); + D.dollarAmount = 1.11m; + D.active = true; + D.roles = 0; + D.notes = "This record will match in notes and tag: elementary my dear aardvark"; + D.tags=TagsArray; + + ApiResponse a = await Util.PostAsync("Widget", await Util.GetTokenAsync("manager", "l3tm3in"), D.ToString()); + Util.ValidateDataReturnResponseOk(a); + long MatchWidgetInNotesId = a.ObjectResponse["data"]["id"].Value(); + + - // //CREATE A WIDGET - // dynamic D = new JObject(); - // D.name = Util.Uniquify("TAG search test WIDGET TAG AND PHRASE"); - // D.dollarAmount = 1.11m; - // D.active = true; - // D.roles = 0; - // D.notes = "This record will match in notes and tag: elementary my dear aardvark"; + //CREATE A WIDGET WITH TAG BUT NOT SEARCH PHRASE + D = new JObject(); + D.name = Util.Uniquify("TAG search test WIDGET TAG ONLY"); + D.dollarAmount = 1.11m; + D.active = true; + D.roles = 0; + D.notes = "This record will match in tag but no search phrase"; + D.tags=TagsArray; - // a = await Util.PostAsync("Widget", await Util.GetTokenAsync("manager", "l3tm3in"), D.ToString()); - // Util.ValidateDataReturnResponseOk(a); - // long MatchWidgetInNotesId = a.ObjectResponse["data"]["id"].Value(); + a = await Util.PostAsync("Widget", await Util.GetTokenAsync("manager", "l3tm3in"), D.ToString()); + Util.ValidateDataReturnResponseOk(a); + long NoPhraseMatchWidgetInTagId = a.ObjectResponse["data"]["id"].Value(); - // //TAG the widget with the fresh tag - // D = new JObject(); - // D.tagId = TagId; - // D.tagToObjectId = MatchWidgetInNotesId; - // D.tagToObjectType = 2;//widget - // a = await Util.PostAsync("TagMap", await Util.GetTokenAsync("BizAdminFull"), D.ToString()); - // Util.ValidateDataReturnResponseOk(a); + + + //CREATE FIRST TEST USER WITH PHRASE IN NAME BUT NO TAG + D = new JObject(); + D.name = Util.Uniquify("Wildcard contains search NAME elementary aardvark Test User"); + D.notes = "This user has the match in it's name but no tag match"; + D.ownerId = 1L; + D.active = true; + D.login = Util.Uniquify("LOGIN"); + D.password = Util.Uniquify("PASSWORD"); + D.roles = 0;//norole + D.localeId = 1;//random locale + D.userType = 3;//non scheduleable + + a = await Util.PostAsync("User", await Util.GetTokenAsync("manager", "l3tm3in"), D.ToString()); + Util.ValidateDataReturnResponseOk(a); + long MatchUserInNameId = a.ObjectResponse["data"]["id"].Value(); - // //CREATE A WIDGET WITH TAG BUT NOT SEARCH PHRASE - // D = new JObject(); - // D.name = Util.Uniquify("TAG search test WIDGET TAG ONLY"); - // D.dollarAmount = 1.11m; - // D.active = true; - // D.roles = 0; - // D.notes = "This record will match in tag but no search phrase"; + //Now see if can find those objects with a phrase search + dynamic SearchParameters = new JObject(); - // a = await Util.PostAsync("Widget", await Util.GetTokenAsync("manager", "l3tm3in"), D.ToString()); - // Util.ValidateDataReturnResponseOk(a); - // long NoPhraseMatchWidgetInTagId = a.ObjectResponse["data"]["id"].Value(); + SearchParameters.phrase = TEST_SEARCH_PHRASE; + SearchParameters.nameOnly = false; + SearchParameters.typeOnly = 0;//no type + //product.Tags = new JArray("Real", "OnSale"); + SearchParameters.tags = new JArray(new long[] { TagId }); + a = await Util.PostAsync("Search", await Util.GetTokenAsync("manager", "l3tm3in"), SearchParameters.ToString()); + Util.ValidateDataReturnResponseOk(a); - // //TAG the tag only no phrase with the fresh tag - // D = new JObject(); - // D.tagId = TagId; - // D.tagToObjectId = NoPhraseMatchWidgetInTagId; - // D.tagToObjectType = 2;//widget + //Now validate the return list + ((JArray)a.ObjectResponse["data"]["searchResults"]).Count.Should().BeGreaterOrEqualTo(1); - // a = await Util.PostAsync("TagMap", await Util.GetTokenAsync("BizAdminFull"), D.ToString()); - // Util.ValidateDataReturnResponseOk(a); + //Turn the list into an array of id's + var v = ((JArray)a.ObjectResponse["data"]["searchResults"]); + List MatchingIdList = new List(); + foreach (JObject j in v) + { + MatchingIdList.Add(j["id"].Value()); + } + //Ensure the expected items are returned + MatchingIdList.Should().Contain(MatchWidgetInNotesId, "ShouldContainMatchWidgetInNotesId"); + MatchingIdList.Should().NotContain(MatchUserInNameId, "ShouldNotContainMatchUserInNameId"); + MatchingIdList.Should().NotContain(NoPhraseMatchWidgetInTagId, "ShouldNotContainNoPhraseMatchWidgetInTagId"); - // //CREATE FIRST TEST USER WITH PHRASE IN NAME BUT NO TAG - // D = new JObject(); - // D.name = Util.Uniquify("Wildcard contains search NAME elementary aardvark Test User"); - // D.notes = "This user has the match in it's name but no tag match"; - // D.ownerId = 1L; - // D.active = true; - // D.login = Util.Uniquify("LOGIN"); - // D.password = Util.Uniquify("PASSWORD"); - // D.roles = 0;//norole - // D.localeId = 1;//random locale - // D.userType = 3;//non scheduleable - - // a = await Util.PostAsync("User", await Util.GetTokenAsync("manager", "l3tm3in"), D.ToString()); - // Util.ValidateDataReturnResponseOk(a); - // long MatchUserInNameId = a.ObjectResponse["data"]["id"].Value(); - - - // //Now see if can find those objects with a phrase search - // dynamic SearchParameters = new JObject(); - - // SearchParameters.phrase = TEST_SEARCH_PHRASE; - // SearchParameters.nameOnly = false; - // SearchParameters.typeOnly = 0;//no type - // //product.Tags = new JArray("Real", "OnSale"); - // SearchParameters.tags = new JArray(new long[] { TagId }); - // a = await Util.PostAsync("Search", await Util.GetTokenAsync("manager", "l3tm3in"), SearchParameters.ToString()); - // Util.ValidateDataReturnResponseOk(a); - - // //Now validate the return list - // ((JArray)a.ObjectResponse["data"]["searchResults"]).Count.Should().BeGreaterOrEqualTo(1); - - // //Turn the list into an array of id's - // var v = ((JArray)a.ObjectResponse["data"]["searchResults"]); - // List MatchingIdList = new List(); - // foreach (JObject j in v) - // { - // MatchingIdList.Add(j["id"].Value()); - // } - - // //Ensure the expected items are returned - // MatchingIdList.Should().Contain(MatchWidgetInNotesId, "ShouldContainMatchWidgetInNotesId"); - // MatchingIdList.Should().NotContain(MatchUserInNameId, "ShouldNotContainMatchUserInNameId"); - // MatchingIdList.Should().NotContain(NoPhraseMatchWidgetInTagId, "ShouldNotContainNoPhraseMatchWidgetInTagId"); - - // }//eot + }//eot