diff --git a/devdocs/todo.txt b/devdocs/todo.txt index 05280c73..d76b088f 100644 --- a/devdocs/todo.txt +++ b/devdocs/todo.txt @@ -1,6 +1,6 @@ # TODO (J.F.C. - Just fucking code it already) -Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOiIxNTM1NDc4Mzc5IiwiZXhwIjoiMTUzODA3MDM3OSIsImlzcyI6IkF5YU5vdmEiLCJpZCI6IjEifQ.VP0amo0xXRaXrMSwkiowjJh-u3tWJVnxvzB5PxJIeiM +Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOiIxNTM4NDI3MDExIiwiZXhwIjoiMTU0MTAxOTAxMSIsImlzcyI6IkF5YU5vdmEiLCJpZCI6IjEifQ.Uwcc1o5l4S6nrEhIlrUJBcR7k2PH8LiFNCWfeWGDNU4 ## IMMEDIATE ITEMS @@ -27,9 +27,10 @@ Once that is done then can steam ahead on the biz objects but until I have the c IMMEDIATE ITEMS: ================ -Happy Monday Radiant Maiden! - - Search and search text indexing + + - Search and search text indexing + - https://gist.github.com/ruckus/5718112 - Can it be sped up with the huge dataset test - Right now a full unconstrained search done *while running all tests at once* is taking 38 seconds!! diff --git a/server/AyaNova/biz/Search.cs b/server/AyaNova/biz/Search.cs index 7bc8141e..a3a6b158 100644 --- a/server/AyaNova/biz/Search.cs +++ b/server/AyaNova/biz/Search.cs @@ -10,6 +10,7 @@ using Microsoft.Extensions.Logging; using Microsoft.EntityFrameworkCore; using AyaNova.Util; using AyaNova.Models; +using System.Diagnostics; namespace AyaNova.Biz @@ -21,8 +22,7 @@ namespace AyaNova.Biz /* ISSUES: - none at the moment - This is pretty fast so going to put a pin in it for now and if required later can sort it out. + Search of big data a little slow, attempt to tweak indices */ @@ -148,11 +148,14 @@ namespace AyaNova.Biz //List holder for matching dictionary ID's List DictionaryMatches = new List(); + + //GET LIST OF DICTIONARY ID'S THAT MATCH REGULAR SEARCH TERMS if (RegularSearchTerms.Count > 0) DictionaryMatches = await ct.SearchDictionary.Where(m => RegularSearchTerms.Contains(m.Word)).Select(m => m.Id).ToListAsync(); + //GET LIST OF DICTIONARY ID'S THAT MATCH WILDCARD SEARCH TERMS if (WildCardSearchTerms.Count > 0) { @@ -191,9 +194,11 @@ namespace AyaNova.Biz q = q.Where(m => m.ObjectType == searchParameters.TypeOnly); + //Find the records that have the search terms in searchkey var SearchMatches = q.GroupBy(x => new { x.ObjectType, x.ObjectId }).Select(x => new { ObjectId = x.Key.ObjectId, ObjectType = x.Key.ObjectType, ObjectCount = x.LongCount() }); + //PUT THE RESULTS INTO MATCHING OBJECTS LIST foreach (var SearchMatch in SearchMatches) { @@ -203,7 +208,8 @@ namespace AyaNova.Biz } - //IF TAGS SPECIFIED + //IF TAGS SPECIFIED + //BUGBUG: If no valid tags provided, i.e. a single tag of type or id 0 then can skip if (searchParameters.Tags.Count > 0) { //get a count of the search tags (used by both paths below) @@ -320,13 +326,26 @@ namespace AyaNova.Biz //MAXIMUM RESULTS FILTER //The theory is that it should be filtered BEFORE sorting so that you get the most random collection of results //As the results are not ranked so... + var watch = new System.Diagnostics.Stopwatch();//###################### PROFILING + watch.Start();//###################### PROFILING + //BUGBUG: THIS is what is taking all the time in the queries FFS if (searchParameters.MaxResults > 0)//0 = all results MatchingObjects = MatchingObjects.Take(searchParameters.MaxResults).ToList(); + watch.Stop();//###################### PROFILING + var TimeToMaximumResultsFilter = watch.ElapsedMilliseconds;//###################### PROFILING + watch.Reset(); + watch.Start();//###################### PROFILING //Sort and group the matching objects list in return order //Customer.OrderBy(c => c.LastName).ThenBy(c => c.FirstName) var OrderedMatchingObjects = MatchingObjects.OrderBy(x => x.ObjectType).ThenByDescending(x => x.ObjectId); + watch.Stop();//###################### PROFILING + var TimeToOrderMatchingObjects = watch.ElapsedMilliseconds;//###################### PROFILING + watch.Reset(); + + + watch.Start();//###################### PROFILING //Build the return list from the remaining matching objects list foreach (AyaTypeId i in OrderedMatchingObjects) { @@ -337,6 +356,10 @@ namespace AyaNova.Biz ResultList.Add(SR); } + watch.Stop();//###################### PROFILING + var TimeToBuildSearchResultReturnList = watch.ElapsedMilliseconds;//###################### PROFILING + + return ResultList; } diff --git a/server/AyaNova/util/AySchema.cs b/server/AyaNova/util/AySchema.cs index 9d608b6d..43b8423e 100644 --- a/server/AyaNova/util/AySchema.cs +++ b/server/AyaNova/util/AySchema.cs @@ -23,7 +23,7 @@ namespace AyaNova.Util private const int DESIRED_SCHEMA_LEVEL = 9; internal const long EXPECTED_COLUMN_COUNT = 99; - internal const long EXPECTED_INDEX_COUNT = 21; + internal const long EXPECTED_INDEX_COUNT = 22; //!!!!WARNING: BE SURE TO UPDATE THE DbUtil::PrepareDatabaseForSeeding WHEN NEW TABLES ADDED!!!! ///////////////////////////////////////////////////////////////// diff --git a/test/raven-integration/Search/SearchOps.cs b/test/raven-integration/Search/SearchOps.cs index 7db4daca..0d1a8406 100644 --- a/test/raven-integration/Search/SearchOps.cs +++ b/test/raven-integration/Search/SearchOps.cs @@ -517,6 +517,20 @@ namespace raven_integration TimeToSearch.Should().BeLessThan(38427, "Unconstrained big data search should not be too slow"); //Fastest is 17227 ms with 14143 results + /* + Indexing improvements testing runs. + + BEFORE ATTEMPTING TO IMPROVE INDEXES + 18651, 22157, 17501, 18221, 17342 = avg: 18774 + + FIRST ATTEMPT on asearchkey.wordid created index in pgadmin: CREATE INDEX searchkey_idxwordid ON public.asearchkey USING btree (wordid ASC NULLS LAST) TABLESPACE pg_default; + 18647, 17137, 16836, 17379, 17637 = avg: 17527 + Not a huge change, but it is positive + + Adding more indexes... + + + */ }//eot