devdocs/todo.txt

@@ -1,6 +1,6 @@
 # TODO (J.F.C. - Just fucking code it already)
 
-Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOiIxNTM1NDc4Mzc5IiwiZXhwIjoiMTUzODA3MDM3OSIsImlzcyI6IkF5YU5vdmEiLCJpZCI6IjEifQ.VP0amo0xXRaXrMSwkiowjJh-u3tWJVnxvzB5PxJIeiM
+Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOiIxNTM4NDI3MDExIiwiZXhwIjoiMTU0MTAxOTAxMSIsImlzcyI6IkF5YU5vdmEiLCJpZCI6IjEifQ.Uwcc1o5l4S6nrEhIlrUJBcR7k2PH8LiFNCWfeWGDNU4
 
 
 ## IMMEDIATE ITEMS
@@ -27,9 +27,10 @@ Once that is done then can steam ahead on the biz objects but until I have the c
 
 IMMEDIATE ITEMS:
 ================
-Happy Monday Radiant Maiden!
 
-	- Search and search text indexing					
+
+	- Search and search text indexing		
+	- https://gist.github.com/ruckus/5718112			
 		
 		- Can it be sped up with the huge dataset test
 			- Right now a full unconstrained search done *while running all tests at once* is taking 38 seconds!!

server/AyaNova/biz/Search.cs

@@ -10,6 +10,7 @@ using Microsoft.Extensions.Logging;
 using Microsoft.EntityFrameworkCore;
 using AyaNova.Util;
 using AyaNova.Models;
+using System.Diagnostics;
 
 
 namespace AyaNova.Biz
@@ -21,8 +22,7 @@ namespace AyaNova.Biz
 
         /*
         ISSUES:
-           none at the moment
-           This is pretty fast so going to put a pin in it for now and if required later can sort it out.
+           Search of big data a little slow, attempt to tweak indices
 
         
          */
@@ -148,11 +148,14 @@ namespace AyaNova.Biz
             //List holder for matching dictionary ID's
             List<long> DictionaryMatches = new List<long>();
 
+
+
             //GET LIST OF DICTIONARY ID'S THAT MATCH REGULAR SEARCH TERMS
             if (RegularSearchTerms.Count > 0)
                 DictionaryMatches = await ct.SearchDictionary.Where(m => RegularSearchTerms.Contains(m.Word)).Select(m => m.Id).ToListAsync();
 
 
+
             //GET LIST OF DICTIONARY ID'S THAT MATCH WILDCARD SEARCH TERMS          
             if (WildCardSearchTerms.Count > 0)
             {
@@ -191,9 +194,11 @@ namespace AyaNova.Biz
                 q = q.Where(m => m.ObjectType == searchParameters.TypeOnly);
 
 
+
             //Find the records that have the search terms in searchkey
             var SearchMatches = q.GroupBy(x => new { x.ObjectType, x.ObjectId }).Select(x => new { ObjectId = x.Key.ObjectId, ObjectType = x.Key.ObjectType, ObjectCount = x.LongCount() });
 
+
             //PUT THE RESULTS INTO MATCHING OBJECTS LIST
             foreach (var SearchMatch in SearchMatches)
             {
@@ -203,7 +208,8 @@ namespace AyaNova.Biz
             }
 
 
-            //IF TAGS SPECIFIED          
+            //IF TAGS SPECIFIED   
+            //BUGBUG: If no valid tags provided, i.e. a single tag of type or id 0 then can skip      
             if (searchParameters.Tags.Count > 0)
             {
                 //get a count of the search tags (used by both paths below)
@@ -320,13 +326,26 @@ namespace AyaNova.Biz
             //MAXIMUM RESULTS FILTER
             //The theory is that it should be filtered BEFORE sorting so that you get the most random collection of results
             //As the results are not ranked so...
+            var watch = new System.Diagnostics.Stopwatch();//###################### PROFILING
+            watch.Start();//###################### PROFILING
+            //BUGBUG: THIS is what is taking all the time in the queries FFS
             if (searchParameters.MaxResults > 0)//0 = all results
                 MatchingObjects = MatchingObjects.Take(searchParameters.MaxResults).ToList();
+            watch.Stop();//###################### PROFILING
+            var TimeToMaximumResultsFilter = watch.ElapsedMilliseconds;//###################### PROFILING
+            watch.Reset();
 
+            watch.Start();//###################### PROFILING
             //Sort and group the matching objects list in return order
             //Customer.OrderBy(c => c.LastName).ThenBy(c => c.FirstName)
             var OrderedMatchingObjects = MatchingObjects.OrderBy(x => x.ObjectType).ThenByDescending(x => x.ObjectId);
 
+            watch.Stop();//###################### PROFILING
+            var TimeToOrderMatchingObjects = watch.ElapsedMilliseconds;//###################### PROFILING
+            watch.Reset();
+
+
+            watch.Start();//###################### PROFILING
             //Build the return list from the remaining matching objects list           
             foreach (AyaTypeId i in OrderedMatchingObjects)
             {
@@ -337,6 +356,10 @@ namespace AyaNova.Biz
                 ResultList.Add(SR);
             }
 
+            watch.Stop();//###################### PROFILING
+            var TimeToBuildSearchResultReturnList = watch.ElapsedMilliseconds;//###################### PROFILING
+
+
             return ResultList;
         }
 

server/AyaNova/util/AySchema.cs

@@ -23,7 +23,7 @@ namespace AyaNova.Util
         private const int DESIRED_SCHEMA_LEVEL = 9;
 
         internal const long EXPECTED_COLUMN_COUNT = 99;
-        internal const long EXPECTED_INDEX_COUNT = 21;
+        internal const long EXPECTED_INDEX_COUNT = 22;
         //!!!!WARNING: BE SURE TO UPDATE THE DbUtil::PrepareDatabaseForSeeding WHEN NEW TABLES ADDED!!!!
 
         /////////////////////////////////////////////////////////////////

test/raven-integration/Search/SearchOps.cs

@@ -517,6 +517,20 @@ namespace raven_integration
             
             TimeToSearch.Should().BeLessThan(38427, "Unconstrained big data search should not be too slow");
             //Fastest is 17227 ms with 14143 results
+            /*
+            Indexing improvements testing runs.
+
+            BEFORE ATTEMPTING TO IMPROVE INDEXES
+            18651, 22157, 17501, 18221, 17342 = avg: 18774
+
+            FIRST ATTEMPT on asearchkey.wordid created index in pgadmin: CREATE INDEX searchkey_idxwordid ON public.asearchkey USING btree (wordid ASC NULLS LAST) TABLESPACE pg_default;
+            18647, 17137, 16836, 17379, 17637 = avg: 17527
+            Not a huge change, but it is positive
+
+            Adding more indexes...
+            
+
+             */
 
         }//eot