diff --git a/devdocs/specs/core-search.txt b/devdocs/specs/core-search.txt index 16f4e98d..a10cc47a 100644 --- a/devdocs/specs/core-search.txt +++ b/devdocs/specs/core-search.txt @@ -85,7 +85,7 @@ asearchkey - wordid (fk on asearchdictionary.id) - objectid (long id of source object) - objecttype (AyaType as int of source object) - - inname (bool indicates the search word was in the name of the object) + diff --git a/docs/8.0/ayanova/docs/form-home-search.md b/docs/8.0/ayanova/docs/form-home-search.md index 852d7432..ac6221d5 100644 --- a/docs/8.0/ayanova/docs/form-home-search.md +++ b/docs/8.0/ayanova/docs/form-home-search.md @@ -1,3 +1,16 @@ # HOME-SEARCH Placeholder -This is a placeholder page for sections that are not written yet +Explain search phrase entry, stopwords etc + +Searches are contains by default, case insensitive always. +Wildcards are supported for start and end of words but no point in doing both ends for contains as that is the default +multiple space seperated phrases are supported and are always inclusive (AND), i.e. "cotton fish" finds only records with both cotton and fish in them + +All text on object is search indexed including tags. +In cases where there is a identifying number on an object such as an ID or serial number that is also indexed for search, however currency amounts are not indexed + +To save bandwidth excerpts are not automatically retrieved but only on demand when user requests for a search result + + + +There is no facility or setting to search case sensitive, it's just not an option with text indexing we have and differs from picklist which does have a server setting to control that. diff --git a/server/AyaNova/biz/Search.cs b/server/AyaNova/biz/Search.cs index 6aa1607d..185c5422 100644 --- a/server/AyaNova/biz/Search.cs +++ b/server/AyaNova/biz/Search.cs @@ -29,7 +29,7 @@ namespace AyaNova.Biz - Search phrase (with wildcard support) - Can be empty if tags are specified, no tags and no phrase is an error condition - ObjectType: only return results for objects of this type - - InName: flag that indicates only search in names + @@ -61,7 +61,7 @@ namespace AyaNova.Biz public class SearchRequestParameters { public string Phrase { get; set; } - public bool NameOnly { get; set; } + public AyaType TypeOnly { get; set; } //Note: maxresults of 0 will get all results @@ -69,7 +69,7 @@ namespace AyaNova.Biz public SearchRequestParameters() { - NameOnly = false; + TypeOnly = AyaType.NoType; MaxResults = 500; } @@ -188,7 +188,7 @@ namespace AyaNova.Biz } } - //SEARCH SEARCHKEY FOR MATCHING WORDS AND OPTIONALLY TYPE AND INNAME + //SEARCH SEARCHKEY FOR MATCHING WORDS AND OPTIONALLY TYPE var TotalSearchTermsToMatch = PreWildCardedSearchTerms.Count + SearchTerms.Count; // var TestRawMatches = await ct.SearchKey.Where(x => DictionaryMatches.Contains(x.WordId)).ToListAsync(); @@ -196,9 +196,6 @@ namespace AyaNova.Biz //Build search query based on searchParameters var q = ct.SearchKey.Distinct().Where(x => DictionaryMatches.Contains(x.WordId)); - //In name? - if (searchParameters.NameOnly) - q = q.Where(m => m.InName == true); //Of type? if (searchParameters.TypeOnly != AyaType.NoType) @@ -218,126 +215,21 @@ namespace AyaNova.Biz } - #region TAGS DEPRECATED - // //--------------- TAGS ARE DEPRECATED AS BEING SOMETHING YOU CAN SEARCH FOR SEPARATELY (INTAGS) INSTEAD THEY ARE TREATED AS PART OF THE TEXT OF THE OBJECT like any other - // //IF TAGS SPECIFIED - // if (searchParameters.Tags.Count > 0) - // { - // //get a count of the search tags (used by both paths below) - // var SearchTagCount = searchParameters.Tags.Count; - - // if (string.IsNullOrWhiteSpace(searchParameters.Phrase)) - // { - - // #region TAGS ONLY SEARCH (NO PHRASE) ALL FULL MATCHES ARE INCLUSIVE - // Dictionary TagCounts = new Dictionary(); - - // //QUERY FOR ALL TAGMAPS THAT MATCH OBJECT TYPE AND ID FOR EVERY TAG SPECIFIED (UNION) - // //var tagmatches= await ct.TagMap.Where(m => ).Select(m => m.Id).ToListAsync(); - // //ct.TagMap.Where(n => n.Tags.Count(t => tags.Contains(t.DisplayName)) == tags.Count) - - // //algorithm: - // //1) get counts for each tag specified from tagmap, if any are zero then none match and can bail early - // foreach (long SearchTagId in searchParameters.Tags) - // { - // var MatchTagCount = await ct.TagMap.Where(m => m.TagId == SearchTagId).LongCountAsync(); - // //zero tags matching here at any point means no results for the entire search and we can bail - // if (MatchTagCount == 0) - // { - // //return empty resultlist - // return ReturnObject; - // } - - // //Save the matching count - // TagCounts.Add(SearchTagId, MatchTagCount); - // } - - // //2) find smallest count match so we are working with the shortest list first - // var ShortestMatchingTag = TagCounts.OrderBy(x => x.Value).First().Key; - - // //3) Generate the shortlist of items that match the shortest tag list - // var ShortList = await ct.TagMap.Where(x => x.TagId == ShortestMatchingTag).ToListAsync(); - - // //4) Iterate the shortlist and see if each item matches all other tags specified if it does then put it into the matching objects list for return - - // //Iterate shortlist - // foreach (TagMap t in ShortList) - // { - // var matchCount = 1; - // //Iterate requested tags - // foreach (long TagId in searchParameters.Tags) - // { - // //skipping already matched shortest tag - // if (TagId != ShortestMatchingTag) - // { - // //Ok, does this object have this tag? - // bool HasTag = await ct.TagMap.Where(x => x.TagToObjectId == t.TagToObjectId && x.TagToObjectType == t.TagToObjectType && x.TagId == TagId).AnyAsync(); - // if (HasTag) - // matchCount++; - // } - // } - // //does it match all tags? - // if (matchCount == SearchTagCount) - // { - // //yes, add it to the results - // MatchingObjects.Add(new AyaTypeId(t.TagToObjectType, t.TagToObjectId)); - // } - // } - // #endregion - - - // } - // else - // { - // #region TAGS PLUS PHRASE SEARCH WITH NON MATCHING TAGS EXCLUSIVE - // //list to hold temporary matches - // List TagMatchingObjects = new List(); - - // //LOOP THROUGH MATCHING OBJECTS LIST - // foreach (AyaTypeId i in MatchingObjects) - // { - // var matchCount = await ct.TagMap.Where(x => x.TagToObjectId == i.ObjectId && x.TagToObjectType == i.ObjectType && searchParameters.Tags.Contains(x.TagId)).LongCountAsync(); - // if (matchCount == SearchTagCount) - // { - // TagMatchingObjects.Add(i); - // } - - // } - - // //Ok here we have all the MatchingObjects that had all the tags in the TagMatchingObjects list so that's actually now our defacto return list - // MatchingObjects = TagMatchingObjects; - - - // #endregion - - // } - // } - - //--------------------- END TAGS ------------- - #endregion tags DEPRECATED - - //REMOVE ANY ITEMS THAT USER IS NOT PERMITTED TO READ - //If it's a name only search then all is allowed - //If it's not a name only search then rights need to be checked for full read because even if it's just a tags search that's part of the full record of the object - //Note: I have decided in the interests of simplicity that even if the result was only found in the name, the user still needs full rights to read the object if the type of search - //was not InNameOnly type. This greatly simplifies processing. - if (!searchParameters.NameOnly) + //list to hold temporary matches + List CanReadMatchingObjects = new List(); + foreach (AyaTypeId t in MatchingObjects) { - //list to hold temporary matches - List CanReadMatchingObjects = new List(); - foreach (AyaTypeId t in MatchingObjects) + if (AyaNova.Api.ControllerHelpers.Authorized.HasReadFullRole(currentUserRoles, t.ObjectType)) { - if (AyaNova.Api.ControllerHelpers.Authorized.HasReadFullRole(currentUserRoles, t.ObjectType)) - { - CanReadMatchingObjects.Add(t); - } + CanReadMatchingObjects.Add(t); } - - //Ok, we're here with the list of allowable objects which is now the master matching objects list so... - MatchingObjects = CanReadMatchingObjects; } + //Ok, we're here with the list of allowable objects which is now the master matching objects list so... + MatchingObjects = CanReadMatchingObjects; + + //TOTAL RESULTS //we have the total results here so set accordingly @@ -393,19 +285,19 @@ namespace AyaNova.Biz public long TranslationId { get; set; } public long ObjectId { get; set; } public AyaType ObjectType { get; set; } - public string Name { get; set; } + // public string Name { get; set; } public List Words { get; set; } - public SearchIndexProcessObjectParameters(long translationId, long objectID, AyaType objectType, string name) + public SearchIndexProcessObjectParameters(long translationId, long objectID, AyaType objectType) { Words = new List(); TranslationId = translationId; ObjectId = objectID; ObjectType = objectType; - Name = name; + // Name = name; } @@ -499,19 +391,17 @@ namespace AyaNova.Biz //BREAK OBJECT TEXT STRINGS INTO KEYWORD LIST List KeyWordList = await BreakAsync(p.TranslationId, p.Words); - //BREAK NAME STRING - List NameKeyWordList = await BreakAsync(p.TranslationId, p.Name); - //EARLY EXIT IF NO KEYWORDS OR NAME RECORD OR TAGS TO PROCESS - if (KeyWordList.Count == 0 && string.IsNullOrWhiteSpace(p.Name)) + //EARLY EXIT IF NO KEYWORDS TO PROCESS + if (KeyWordList.Count == 0) { return; } //BUILD A LIST OF MatchingDictionaryEntry items FOR THE MATCHING WORDS - List MatchingKeywordIdList = new List(); + List MatchingKeywordIdList = new List(); //ITERATE ALL THE KEYWORDS, SEARCH IN THE SEARCHDICTIONARY TABLE AND COLLECT ID'S OF ANY PRE-EXISTING IN DB KEYWORDS var ExistingKeywordMatches = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(m => KeyWordList.Contains(m.Word)).ToDictionaryAsync(m => m.Id, m => m.Word); @@ -527,10 +417,7 @@ WHERE a.word IN ('eos', 'quia', 'voluptate', 'delectus', 'sapiente', 'omnis', 's //Put the matching keyword ID's into the list foreach (KeyValuePair K in ExistingKeywordMatches) { - //Name or regular word match? - bool IsName = NameKeyWordList.Contains(K.Value); - MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = K.Key, InName = IsName }); - + MatchingKeywordIdList.Add(K.Key); } //-------- START CRITICAL SECTION ----------- @@ -621,7 +508,7 @@ cache or provide directly the translation to save time repeatedly fetching it wh //------- //Add to matching keywords - MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = NewWord.Id, InName = NameKeyWordList.Contains(KeyWord) }); + MatchingKeywordIdList.Add(NewWord.Id); //------- //It exists now @@ -646,7 +533,7 @@ cache or provide directly the translation to save time repeatedly fetching it wh var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(x => x.Word == KeyWord).FirstOrDefaultAsync(); if (SearchDictionaryMatchFoundInDB != null) { - MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = SearchDictionaryMatchFoundInDB.Id, InName = NameKeyWordList.Contains(KeyWord) }); + MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id); //It exists now ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word); } @@ -677,9 +564,9 @@ cache or provide directly the translation to save time repeatedly fetching it wh //CREATE THE SEARCHKEY RECORDS FOR ALL THE KEYWORDS var NewSearchKeyList = new List(); - foreach (MatchingDictionaryEntry E in MatchingKeywordIdList) + foreach (long E in MatchingKeywordIdList) { - NewSearchKeyList.Add(new SearchKey() { WordId = E.DictionaryId, InName = E.InName, ObjectId = p.ObjectId, ObjectType = p.ObjectType }); + NewSearchKeyList.Add(new SearchKey() { WordId = E, ObjectId = p.ObjectId, ObjectType = p.ObjectType }); } var CtSearchKeyAdd = ServiceProviderProvider.DBContext; await CtSearchKeyAdd.SearchKey.AddRangeAsync(NewSearchKeyList); @@ -689,19 +576,6 @@ cache or provide directly the translation to save time repeatedly fetching it wh }//eoc - //Class to hold temporary list of matching id - public class MatchingDictionaryEntry - { - public bool InName { get; set; } - - public long DictionaryId { get; set; } - public MatchingDictionaryEntry() - { - InName = false; - - DictionaryId = -1; - } - } #endregion diff --git a/server/AyaNova/biz/UserBiz.cs b/server/AyaNova/biz/UserBiz.cs index 2675dcfc..d6fb10c9 100644 --- a/server/AyaNova/biz/UserBiz.cs +++ b/server/AyaNova/biz/UserBiz.cs @@ -220,7 +220,7 @@ namespace AyaNova.Biz private async Task SearchIndexAsync(User obj, bool isNew) { //SEARCH INDEXING - var SearchParams = new Search.SearchIndexProcessObjectParameters(UserTranslationId, obj.Id, BizType, obj.Name); + var SearchParams = new Search.SearchIndexProcessObjectParameters(UserTranslationId, obj.Id, BizType); SearchParams.AddText(obj.Notes).AddText(obj.Name).AddText(obj.EmployeeNumber).AddText(obj.Tags).AddCustomFields(obj.CustomFields); if (isNew) diff --git a/server/AyaNova/biz/WidgetBiz.cs b/server/AyaNova/biz/WidgetBiz.cs index 92d606ae..2730a1ad 100644 --- a/server/AyaNova/biz/WidgetBiz.cs +++ b/server/AyaNova/biz/WidgetBiz.cs @@ -201,7 +201,7 @@ namespace AyaNova.Biz private async Task SearchIndexAsync(Widget obj, bool isNew) { //SEARCH INDEXING - var SearchParams = new Search.SearchIndexProcessObjectParameters(UserTranslationId, obj.Id, BizType, obj.Name); + var SearchParams = new Search.SearchIndexProcessObjectParameters(UserTranslationId, obj.Id, BizType); SearchParams.AddText(obj.Notes).AddText(obj.Name).AddText(obj.Serial).AddText(obj.Tags).AddCustomFields(obj.CustomFields); diff --git a/server/AyaNova/models/SearchKey.cs b/server/AyaNova/models/SearchKey.cs index 394eff6c..c9c5c6ef 100644 --- a/server/AyaNova/models/SearchKey.cs +++ b/server/AyaNova/models/SearchKey.cs @@ -17,8 +17,8 @@ namespace AyaNova.Models public long ObjectId { get; set; } [Required] public AyaType ObjectType { get; set; } - public bool InName { get; set; } - + + } } diff --git a/server/AyaNova/util/AySchema.cs b/server/AyaNova/util/AySchema.cs index 65d887e3..a4320818 100644 --- a/server/AyaNova/util/AySchema.cs +++ b/server/AyaNova/util/AySchema.cs @@ -22,7 +22,7 @@ namespace AyaNova.Util //!!!!WARNING: BE SURE TO UPDATE THE DbUtil::EmptyBizDataFromDatabaseForSeedingOrImporting WHEN NEW TABLES ADDED!!!! private const int DESIRED_SCHEMA_LEVEL = 10; - internal const long EXPECTED_COLUMN_COUNT = 102; + internal const long EXPECTED_COLUMN_COUNT = 101; internal const long EXPECTED_INDEX_COUNT = 28; //!!!!WARNING: BE SURE TO UPDATE THE DbUtil::EmptyBizDataFromDatabaseForSeedingOrImporting WHEN NEW TABLES ADDED!!!! @@ -152,7 +152,7 @@ namespace AyaNova.Util //Search dictionary words must be unique await ExecQueryAsync("CREATE UNIQUE INDEX asearchdictionary_word_idx ON asearchdictionary (word);"); - await ExecQueryAsync("CREATE TABLE asearchkey (id BIGSERIAL PRIMARY KEY, wordid bigint not null REFERENCES asearchdictionary (id), objectid bigint not null, objecttype integer not null, inname bool not null)"); + await ExecQueryAsync("CREATE TABLE asearchkey (id BIGSERIAL PRIMARY KEY, wordid bigint not null REFERENCES asearchdictionary (id), objectid bigint not null, objecttype integer not null)"); //create translation text tables await ExecQueryAsync("CREATE TABLE atranslation (id BIGSERIAL PRIMARY KEY, name varchar(255) not null, stock bool, cjkindex bool default false)");