.vscode/launch.json

@@ -40,8 +40,8 @@
       "env": {
         "ASPNETCORE_ENVIRONMENT": "Development",
         "AYANOVA_JWT_SECRET": "UNLICENSED5G*QQJ8#bQ7$Xr_@sXfHq4",
-        //"AYANOVA_LOG_LEVEL": "Info",
+        "AYANOVA_LOG_LEVEL": "Info",
-        "AYANOVA_LOG_LEVEL": "Debug",
+        //"AYANOVA_LOG_LEVEL": "Debug",
         "AYANOVA_DEFAULT_LANGUAGE": "en",
         //LOCALE MUST BE en for Integration TESTING
         //"AYANOVA_PERMANENTLY_ERASE_DATABASE": "true",

devdocs/todo.txt

@@ -5,7 +5,13 @@ eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOiIxNTcxODU5OTU0IiwiZXhwIjoiMTU3MjQ
 
 ## IMMEDIATE ITEMS
 
-PERFORMANCE WORK
+///////////////////////////////////////////////////////////////////////////////
+TODO: Search indexing is painfully slow, it accounts for 16 of 22 seconds when creating 500 widgets with full paragraphs of text 
+	- Try to see if it's just one part of the operation by timing it
+	- Re-code it not using EF but directly interacting with the DB
+	- Maybe it's a case for stored procedures or something?
+
+SEARCH INDEXING PERFORMANCE WORK
 Baseline from before doing anything seeding a medium level with full text
 2020-01-21 16:49:17.4662|INFO|Seeder|75 Users seeded in 2279 ms
 2020-01-21 16:49:39.4481|INFO|Seeder|500 Widgets seeded in 21968 ms
@@ -30,16 +36,26 @@ Now going to try the opposite, a *lot* of text 10 paragraphs in both c2 and note
 
 So the quantity of text directly affects the performance, so it's not just some overhead from the query being run, it's the amount of work it needs to do in the queries
 
+THINGS TO TRY:
+Completely alternate methods:
+	- https://stackoverflow.com/a/15089664/8939  Store a Digest of each record with that record then can just search the digests (would mean a search has to traverse all records of every table possibly)
+
+DB INDEX TUNING?
+	- Play with the indexes and see if there is a slowup with an unnecessary index maybe affecting things
+
+Async the keyword processing
+	- Fire off the indexing and return immediately so there would be a bit of time to come into compliance maybe more clashes?
+
+Removing use of EF entirely in search indexing processing in favor of direct sql queries
+
+cache or provide directly the locale to save time repeatedly fetching it when doing bulk ops
+
+///////////////////////////////////////////////////////////////////////////////
 
 
 
 
-TODO: Search indexing is painfully slow, it accounts for 16 of 22 seconds when creating 500 widgets with full paragraphs of text 
+TODO: RUN data list query with debug info on and see if any EF stuff is creeping into it
-	- Try to see if it's just one part of the operation by timing it
-	- Re-code it not using EF but directly interacting with the DB
-	- Maybe it's a case for stored procedures or something?
-
-
 
 
 TODO: DataFilter how to distinguish between filtering on specific ID value or on value column

server/AyaNova/Startup.cs

@@ -134,7 +134,7 @@ namespace AyaNova
             bool LOG_SENSITIVE_DATA = false;
 
 #if (DEBUG)
-            LOG_SENSITIVE_DATA = true;//############################################################################
+            LOG_SENSITIVE_DATA = false;//############################################################################
 
 #endif
 
@@ -449,7 +449,7 @@ namespace AyaNova
             {
                 AyaNova.Core.License.Fetch(apiServerState, dbContext, _newLog);
                 //NOTE: For unit testing make sure the time zone in util is set to the same figure as here to ensure list filter by date tests will work because server is on same page as user in terms of time
-                Util.Seeder.SeedDatabase(Util.Seeder.SeedLevel.SmallOneManShopTrialDataSet, -7);//#############################################################################################
+                Util.Seeder.SeedDatabase(Util.Seeder.SeedLevel.MediumLocalServiceCompanyTrialDataSet, -7);//#############################################################################################
             }
             //TESTING
 #endif

server/AyaNova/biz/Search.cs

@@ -380,10 +380,11 @@ namespace AyaNova.Biz
             public AyaType ObjectType { get; set; }
             public string Name { get; set; }
             public List<string> Words { get; set; }
+            public LocaleWordBreakingData LocaleSearchData { get; set; }
 
 
 
-            public SearchIndexProcessObjectParameters(long localeId, long objectID, AyaType objectType, string name)
+            public SearchIndexProcessObjectParameters(long localeId, long objectID, AyaType objectType, string name, LocaleWordBreakingData localeSearchData = null)
             {
                 Words = new List<string>();
 
@@ -391,6 +392,7 @@ namespace AyaNova.Biz
                 ObjectId = objectID;
                 ObjectType = objectType;
                 Name = name;
+                LocaleSearchData = localeSearchData;
 
             }
 
@@ -525,7 +527,7 @@ WHERE a.word IN ('eos', 'quia', 'voluptate', 'delectus', 'sapiente', 'omnis', 's
             var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###");
 #endif
 
-//TODO: this foreach block needs to add all the words at once in a single range query rather than one word at a time
+            //TODO: this foreach block needs to add all the words at once in a single range query rather than one word at a time
             foreach (string KeyWord in KeyWordList)
             {
                 if (!ExistingKeywordMatches.ContainsValue(KeyWord))
@@ -678,7 +680,8 @@ RETURNING id, xmin;
         }
 
         //Get the current stopwords for the user's locale
-        private static LocaleWordBreakingData GetLocaleSearchData(long localeId, AyContext ct = null)
+        //called in here in this class and also by any bulk ops like seeding etc
+        internal static LocaleWordBreakingData GetLocaleSearchData(long localeId, AyContext ct = null)
         {
             LocaleWordBreakingData LSD = new LocaleWordBreakingData();
             if (ct == null)
@@ -722,35 +725,26 @@ RETURNING id, xmin;
         /// Use Locale setting CJKIndex=true to handle Chinese, Japanese, Korean etc
         /// (languages with no easily identifiable word boundaries as in english)
         /// </summary>       
-        /// 
-        /// <param name="localeId"></param>
-        /// <param name="textStrings">A stringlist of 0 to * strings of text</param>
         /// <returns>List of strings</returns>
-        internal static List<string> Break(long localeId, List<string> textStrings)// params string[] text)
+        internal static List<string> Break(long localeId, List<string> textStrings, LocaleWordBreakingData LocaleSearchData = null)
         {
-            return BreakCore(localeId, false, textStrings);
+            return BreakCore(localeId, false, textStrings, LocaleSearchData);
         }
 
         /// <summary>
         /// 
         /// </summary>        
-        /// <param name="localeId"></param>
+        internal static List<string> Break(long localeId, string textString, LocaleWordBreakingData LocaleSearchData = null)// params string[] text)
-        /// <param name="textString"></param>
-        /// <returns></returns>
-        internal static List<string> Break(long localeId, string textString)// params string[] text)
         {
             List<string> textStrings = new List<string>(1);
             textStrings.Add(textString);
-            return BreakCore(localeId, false, textStrings);
+            return BreakCore(localeId, false, textStrings, LocaleSearchData);
         }
 
         /// <summary>
         /// Used to Process users search phrase and preserve wild
         /// cards entered
         /// </summary>        
-        /// <param name="localeId"></param>
-        /// <param name="searchPhrase"></param>
-        /// <returns></returns>
         internal static List<string> BreakSearchPhrase(long localeId, string searchPhrase)
         {
             List<string> textStrings = new List<string>();
@@ -764,10 +758,12 @@ RETURNING id, xmin;
         /// </summary>
        // public static System.Collections.Generic.List<string> StopList = null;
 
-        internal static List<string> BreakCore(long localeId, bool KeepWildCards, List<string> textStrings)
+        internal static List<string> BreakCore(long localeId, bool KeepWildCards, List<string> textStrings, LocaleWordBreakingData LocaleSearchData = null)
         {
-            //Get stopwords and CJKIndex flag value
+            //For stopwords and CJKIndex flag value
-            LocaleWordBreakingData LocaleSearchData = GetLocaleSearchData(localeId);
+            //if not provided (will be provided by seeder for performance but normally never) then fetch
+            if (LocaleSearchData == null)
+                LocaleSearchData = GetLocaleSearchData(localeId);
             int MAXWORDLENGTH = 255;
             int MINWORDLENGTH = 2;//A word isn't a word unless it's got at least two characters in it
             StringBuilder sbResults = new StringBuilder();

server/AyaNova/biz/WidgetBiz.cs

@@ -65,9 +65,8 @@ namespace AyaNova.Biz
         ////////////////////////////////////////////////////////////////////////////////////////////////
         //CREATE   
 
-
+        //Called from route and also seeder
-        //route linked version for external api access
+        internal Widget Create(Widget inObj, Search.LocaleWordBreakingData LocaleSearchData = null)
-        internal Widget Create(Widget inObj)
         {
             Validate(inObj, null);
             if (HasErrors)
@@ -94,11 +93,11 @@ namespace AyaNova.Biz
 
 
                 //This takes 16 seconds out of 22 when seeding 500 widgets
-                SearchIndex(outObj, true);
+                SearchIndex(outObj, true, LocaleSearchData);
 
 
-               //This takes 2 seconds out of 22 when seeding 500 widgets
+                //This takes 2 seconds out of 22 when seeding 500 widgets
-               TagUtil.ProcessUpdateTagsInRepository(ct, outObj.Tags, null);
+                TagUtil.ProcessUpdateTagsInRepository(ct, outObj.Tags, null);
 
                 return outObj;
             }
@@ -228,10 +227,10 @@ namespace AyaNova.Biz
         }
 
 
-        private void SearchIndex(Widget obj, bool isNew)
+        private void SearchIndex(Widget obj, bool isNew, Search.LocaleWordBreakingData LocaleSearchData=null)
         {
             //SEARCH INDEXING
-            var SearchParams = new Search.SearchIndexProcessObjectParameters(UserLocaleId, obj.Id, BizType, obj.Name);
+            var SearchParams = new Search.SearchIndexProcessObjectParameters(UserLocaleId, obj.Id, BizType, obj.Name, LocaleSearchData);
             SearchParams.AddText(obj.Notes).AddText(obj.Name).AddText(obj.Serial).AddText(obj.Tags).AddCustomFields(obj.CustomFields);
 
 

server/AyaNova/util/Seeder.cs

@@ -17,6 +17,8 @@ namespace AyaNova.Util
         public enum SeedLevel { SmallOneManShopTrialDataSet, MediumLocalServiceCompanyTrialDataSet, LargeCorporateMultiRegionalTrialDataSet, HugeForLoadTest };
         public static int SeededUserCount = 0;
 
+        //cache the locale word breaking data
+        private static Search.LocaleWordBreakingData LocaleSearchData = null;
 
 
         //////////////////////////////////////////////////////
@@ -73,6 +75,12 @@ namespace AyaNova.Util
 
 
 
+                using (var cct = ServiceProviderProvider.DBContext)
+                {
+                    //Get a cached LocaleWordBreakingData to speed up generation 
+                    LocaleSearchData = Search.GetLocaleSearchData(ServerBootConfig.AYANOVA_DEFAULT_LANGUAGE_ID, cct);
+                }
+
                 //Set the time zone of the manager account 
                 using (var cct = ServiceProviderProvider.DBContext)
                 {
@@ -622,7 +630,7 @@ namespace AyaNova.Util
                 //var NewObject = Cached_WidgetBiz.CreateAsync(o).Result;
                 //test without cached widgetbiz
                 WidgetBiz biz = WidgetBiz.GetBizInternal(ServiceProviderProvider.DBContext);
-                var NewObject = biz.Create(o);
+                var NewObject = biz.Create(o, LocaleSearchData);
                 if (NewObject == null)
                 {
                     log.LogError($"Seeder::GenSeedWidget error creating widget {o.Name}\r\n" + biz.GetErrorsAsString());