From b241561fe63946ab49911a7ef912b2615ee45ae2 Mon Sep 17 00:00:00 2001 From: John Cardinal Date: Fri, 24 Jan 2020 19:55:26 +0000 Subject: [PATCH] --- .vscode/launch.json | 4 +-- devdocs/todo.txt | 30 ++++++++++++++++----- server/AyaNova/Startup.cs | 4 +-- server/AyaNova/biz/Search.cs | 46 +++++++++++++++------------------ server/AyaNova/biz/WidgetBiz.cs | 21 +++++++-------- server/AyaNova/util/Seeder.cs | 14 +++++++--- 6 files changed, 69 insertions(+), 50 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index ea665974..17ec1dc0 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -40,8 +40,8 @@ "env": { "ASPNETCORE_ENVIRONMENT": "Development", "AYANOVA_JWT_SECRET": "UNLICENSED5G*QQJ8#bQ7$Xr_@sXfHq4", - //"AYANOVA_LOG_LEVEL": "Info", - "AYANOVA_LOG_LEVEL": "Debug", + "AYANOVA_LOG_LEVEL": "Info", + //"AYANOVA_LOG_LEVEL": "Debug", "AYANOVA_DEFAULT_LANGUAGE": "en", //LOCALE MUST BE en for Integration TESTING //"AYANOVA_PERMANENTLY_ERASE_DATABASE": "true", diff --git a/devdocs/todo.txt b/devdocs/todo.txt index f9918185..f63e74bb 100644 --- a/devdocs/todo.txt +++ b/devdocs/todo.txt @@ -5,7 +5,13 @@ eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOiIxNTcxODU5OTU0IiwiZXhwIjoiMTU3MjQ ## IMMEDIATE ITEMS -PERFORMANCE WORK +/////////////////////////////////////////////////////////////////////////////// +TODO: Search indexing is painfully slow, it accounts for 16 of 22 seconds when creating 500 widgets with full paragraphs of text + - Try to see if it's just one part of the operation by timing it + - Re-code it not using EF but directly interacting with the DB + - Maybe it's a case for stored procedures or something? + +SEARCH INDEXING PERFORMANCE WORK Baseline from before doing anything seeding a medium level with full text 2020-01-21 16:49:17.4662|INFO|Seeder|75 Users seeded in 2279 ms 2020-01-21 16:49:39.4481|INFO|Seeder|500 Widgets seeded in 21968 ms @@ -30,16 +36,26 @@ Now going to try the opposite, a *lot* of text 10 paragraphs in both c2 and note So the quantity of text directly affects the performance, so it's not just some overhead from the query being run, it's the amount of work it needs to do in the queries +THINGS TO TRY: +Completely alternate methods: + - https://stackoverflow.com/a/15089664/8939 Store a Digest of each record with that record then can just search the digests (would mean a search has to traverse all records of every table possibly) + +DB INDEX TUNING? + - Play with the indexes and see if there is a slowup with an unnecessary index maybe affecting things + +Async the keyword processing + - Fire off the indexing and return immediately so there would be a bit of time to come into compliance maybe more clashes? + +Removing use of EF entirely in search indexing processing in favor of direct sql queries + +cache or provide directly the locale to save time repeatedly fetching it when doing bulk ops + +/////////////////////////////////////////////////////////////////////////////// -TODO: Search indexing is painfully slow, it accounts for 16 of 22 seconds when creating 500 widgets with full paragraphs of text - - Try to see if it's just one part of the operation by timing it - - Re-code it not using EF but directly interacting with the DB - - Maybe it's a case for stored procedures or something? - - +TODO: RUN data list query with debug info on and see if any EF stuff is creeping into it TODO: DataFilter how to distinguish between filtering on specific ID value or on value column diff --git a/server/AyaNova/Startup.cs b/server/AyaNova/Startup.cs index edb1203f..5cbfd6d3 100644 --- a/server/AyaNova/Startup.cs +++ b/server/AyaNova/Startup.cs @@ -134,7 +134,7 @@ namespace AyaNova bool LOG_SENSITIVE_DATA = false; #if (DEBUG) - LOG_SENSITIVE_DATA = true;//############################################################################ + LOG_SENSITIVE_DATA = false;//############################################################################ #endif @@ -449,7 +449,7 @@ namespace AyaNova { AyaNova.Core.License.Fetch(apiServerState, dbContext, _newLog); //NOTE: For unit testing make sure the time zone in util is set to the same figure as here to ensure list filter by date tests will work because server is on same page as user in terms of time - Util.Seeder.SeedDatabase(Util.Seeder.SeedLevel.SmallOneManShopTrialDataSet, -7);//############################################################################################# + Util.Seeder.SeedDatabase(Util.Seeder.SeedLevel.MediumLocalServiceCompanyTrialDataSet, -7);//############################################################################################# } //TESTING #endif diff --git a/server/AyaNova/biz/Search.cs b/server/AyaNova/biz/Search.cs index b159386d..ae3bb941 100644 --- a/server/AyaNova/biz/Search.cs +++ b/server/AyaNova/biz/Search.cs @@ -380,10 +380,11 @@ namespace AyaNova.Biz public AyaType ObjectType { get; set; } public string Name { get; set; } public List Words { get; set; } + public LocaleWordBreakingData LocaleSearchData { get; set; } - public SearchIndexProcessObjectParameters(long localeId, long objectID, AyaType objectType, string name) + public SearchIndexProcessObjectParameters(long localeId, long objectID, AyaType objectType, string name, LocaleWordBreakingData localeSearchData = null) { Words = new List(); @@ -391,6 +392,7 @@ namespace AyaNova.Biz ObjectId = objectID; ObjectType = objectType; Name = name; + LocaleSearchData = localeSearchData; } @@ -433,7 +435,7 @@ namespace AyaNova.Biz return this; } - + } @@ -486,7 +488,7 @@ namespace AyaNova.Biz //BREAK NAME STRING List NameKeyWordList = Break(p.LocaleId, p.Name); - + //EARLY EXIT IF NO KEYWORDS OR NAME RECORD OR TAGS TO PROCESS if (KeyWordList.Count == 0 && string.IsNullOrWhiteSpace(p.Name)) @@ -508,7 +510,7 @@ WHERE a.word IN ('eos', 'quia', 'voluptate', 'delectus', 'sapiente', 'omnis', 's */ - + //Put the matching keyword ID's into the list foreach (KeyValuePair K in ExistingKeywordMatches) { @@ -525,7 +527,7 @@ WHERE a.word IN ('eos', 'quia', 'voluptate', 'delectus', 'sapiente', 'omnis', 's var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###"); #endif -//TODO: this foreach block needs to add all the words at once in a single range query rather than one word at a time + //TODO: this foreach block needs to add all the words at once in a single range query rather than one word at a time foreach (string KeyWord in KeyWordList) { if (!ExistingKeywordMatches.ContainsValue(KeyWord)) @@ -678,7 +680,8 @@ RETURNING id, xmin; } //Get the current stopwords for the user's locale - private static LocaleWordBreakingData GetLocaleSearchData(long localeId, AyContext ct = null) + //called in here in this class and also by any bulk ops like seeding etc + internal static LocaleWordBreakingData GetLocaleSearchData(long localeId, AyContext ct = null) { LocaleWordBreakingData LSD = new LocaleWordBreakingData(); if (ct == null) @@ -721,36 +724,27 @@ RETURNING id, xmin; /// /// Use Locale setting CJKIndex=true to handle Chinese, Japanese, Korean etc /// (languages with no easily identifiable word boundaries as in english) - /// - /// - /// - /// A stringlist of 0 to * strings of text + /// /// List of strings - internal static List Break(long localeId, List textStrings)// params string[] text) + internal static List Break(long localeId, List textStrings, LocaleWordBreakingData LocaleSearchData = null) { - return BreakCore(localeId, false, textStrings); + return BreakCore(localeId, false, textStrings, LocaleSearchData); } /// /// - /// - /// - /// - /// - internal static List Break(long localeId, string textString)// params string[] text) + /// + internal static List Break(long localeId, string textString, LocaleWordBreakingData LocaleSearchData = null)// params string[] text) { List textStrings = new List(1); textStrings.Add(textString); - return BreakCore(localeId, false, textStrings); + return BreakCore(localeId, false, textStrings, LocaleSearchData); } /// /// Used to Process users search phrase and preserve wild /// cards entered - /// - /// - /// - /// + /// internal static List BreakSearchPhrase(long localeId, string searchPhrase) { List textStrings = new List(); @@ -764,10 +758,12 @@ RETURNING id, xmin; /// // public static System.Collections.Generic.List StopList = null; - internal static List BreakCore(long localeId, bool KeepWildCards, List textStrings) + internal static List BreakCore(long localeId, bool KeepWildCards, List textStrings, LocaleWordBreakingData LocaleSearchData = null) { - //Get stopwords and CJKIndex flag value - LocaleWordBreakingData LocaleSearchData = GetLocaleSearchData(localeId); + //For stopwords and CJKIndex flag value + //if not provided (will be provided by seeder for performance but normally never) then fetch + if (LocaleSearchData == null) + LocaleSearchData = GetLocaleSearchData(localeId); int MAXWORDLENGTH = 255; int MINWORDLENGTH = 2;//A word isn't a word unless it's got at least two characters in it StringBuilder sbResults = new StringBuilder(); diff --git a/server/AyaNova/biz/WidgetBiz.cs b/server/AyaNova/biz/WidgetBiz.cs index c244e6ca..e6110878 100644 --- a/server/AyaNova/biz/WidgetBiz.cs +++ b/server/AyaNova/biz/WidgetBiz.cs @@ -65,9 +65,8 @@ namespace AyaNova.Biz //////////////////////////////////////////////////////////////////////////////////////////////// //CREATE - - //route linked version for external api access - internal Widget Create(Widget inObj) + //Called from route and also seeder + internal Widget Create(Widget inObj, Search.LocaleWordBreakingData LocaleSearchData = null) { Validate(inObj, null); if (HasErrors) @@ -84,7 +83,7 @@ namespace AyaNova.Biz outObj.CustomFields = JsonUtil.CompactJson(outObj.CustomFields); //Save to db - + //add syncronously and don't save but let the log save //THIS SAVED 1 SECOND OUT OF 22 for seeding 500 widgets ct.Widget.Add(outObj); @@ -94,11 +93,11 @@ namespace AyaNova.Biz //This takes 16 seconds out of 22 when seeding 500 widgets - SearchIndex(outObj, true); - - - //This takes 2 seconds out of 22 when seeding 500 widgets - TagUtil.ProcessUpdateTagsInRepository(ct, outObj.Tags, null); + SearchIndex(outObj, true, LocaleSearchData); + + + //This takes 2 seconds out of 22 when seeding 500 widgets + TagUtil.ProcessUpdateTagsInRepository(ct, outObj.Tags, null); return outObj; } @@ -228,10 +227,10 @@ namespace AyaNova.Biz } - private void SearchIndex(Widget obj, bool isNew) + private void SearchIndex(Widget obj, bool isNew, Search.LocaleWordBreakingData LocaleSearchData=null) { //SEARCH INDEXING - var SearchParams = new Search.SearchIndexProcessObjectParameters(UserLocaleId, obj.Id, BizType, obj.Name); + var SearchParams = new Search.SearchIndexProcessObjectParameters(UserLocaleId, obj.Id, BizType, obj.Name, LocaleSearchData); SearchParams.AddText(obj.Notes).AddText(obj.Name).AddText(obj.Serial).AddText(obj.Tags).AddCustomFields(obj.CustomFields); diff --git a/server/AyaNova/util/Seeder.cs b/server/AyaNova/util/Seeder.cs index 77899c46..449126d3 100644 --- a/server/AyaNova/util/Seeder.cs +++ b/server/AyaNova/util/Seeder.cs @@ -17,6 +17,8 @@ namespace AyaNova.Util public enum SeedLevel { SmallOneManShopTrialDataSet, MediumLocalServiceCompanyTrialDataSet, LargeCorporateMultiRegionalTrialDataSet, HugeForLoadTest }; public static int SeededUserCount = 0; + //cache the locale word breaking data + private static Search.LocaleWordBreakingData LocaleSearchData = null; ////////////////////////////////////////////////////// @@ -73,6 +75,12 @@ namespace AyaNova.Util + using (var cct = ServiceProviderProvider.DBContext) + { + //Get a cached LocaleWordBreakingData to speed up generation + LocaleSearchData = Search.GetLocaleSearchData(ServerBootConfig.AYANOVA_DEFAULT_LANGUAGE_ID, cct); + } + //Set the time zone of the manager account using (var cct = ServiceProviderProvider.DBContext) { @@ -603,7 +611,7 @@ namespace AyaNova.Util //Random but valid enum AuthorizationRoles randomRole = (AuthorizationRoles)values.GetValue(random.Next(values.Length)); o.Roles = randomRole; - + o.Notes = f.Lorem.Sentence(); o.Tags = RandomTags(f); @@ -615,14 +623,14 @@ namespace AyaNova.Util var c3 = f.Random.Int(1, 99999999); var c4 = f.Random.Bool().ToString().ToLowerInvariant(); var c5 = f.Random.Decimal(); - + o.CustomFields = $@"{{c1:""{c1}"",c2:""{c2}"",c3:{c3},c4:{c4},c5:{c5}}}"; //var NewObject = Cached_WidgetBiz.CreateAsync(o).Result; //test without cached widgetbiz WidgetBiz biz = WidgetBiz.GetBizInternal(ServiceProviderProvider.DBContext); - var NewObject = biz.Create(o); + var NewObject = biz.Create(o, LocaleSearchData); if (NewObject == null) { log.LogError($"Seeder::GenSeedWidget error creating widget {o.Name}\r\n" + biz.GetErrorsAsString());