diff --git a/devdocs/todo.txt b/devdocs/todo.txt index f63e74bb..d0feddf0 100644 --- a/devdocs/todo.txt +++ b/devdocs/todo.txt @@ -48,12 +48,18 @@ Async the keyword processing Removing use of EF entirely in search indexing processing in favor of direct sql queries -cache or provide directly the locale to save time repeatedly fetching it when doing bulk ops +cache or provide directly the locale to save time repeatedly fetching it when doing bulk ops!!! + -After doing this 500=21 That's as fast as when I stripped out all the text, what a huge overhead saving right there!: + 2020-01-24 12:00:41.2547|INFO|Seeder|Seeding 500 Widgets.... + 2020-01-24 12:00:51.9138|INFO|Seeder|500 Widgets seeded in 10649 ms /////////////////////////////////////////////////////////////////////////////// - +TODO: Locale search data, there is overhead on every record save due to having to repeatedly fetch this + - While I added a way to seed it without fetching, there should actually just be a cached version for all record saving stored locally + - Fetches in the db are not necessary for this object more than once or on change of locale + - So probably Search code should cache it internally on first fetch TODO: RUN data list query with debug info on and see if any EF stuff is creeping into it diff --git a/server/AyaNova/biz/Search.cs b/server/AyaNova/biz/Search.cs index ae3bb941..96cbbf66 100644 --- a/server/AyaNova/biz/Search.cs +++ b/server/AyaNova/biz/Search.cs @@ -19,7 +19,6 @@ namespace AyaNova.Biz //This class handles word breaking, processing keywords and searching for results public static class Search { - //private static ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("Search"); #region Search and return results @@ -379,20 +378,18 @@ namespace AyaNova.Biz public long ObjectId { get; set; } public AyaType ObjectType { get; set; } public string Name { get; set; } - public List Words { get; set; } - public LocaleWordBreakingData LocaleSearchData { get; set; } + public List Words { get; set; } - public SearchIndexProcessObjectParameters(long localeId, long objectID, AyaType objectType, string name, LocaleWordBreakingData localeSearchData = null) + public SearchIndexProcessObjectParameters(long localeId, long objectID, AyaType objectType, string name) { Words = new List(); LocaleId = localeId; ObjectId = objectID; ObjectType = objectType; - Name = name; - LocaleSearchData = localeSearchData; + Name = name; } @@ -726,19 +723,19 @@ RETURNING id, xmin; /// (languages with no easily identifiable word boundaries as in english) /// /// List of strings - internal static List Break(long localeId, List textStrings, LocaleWordBreakingData LocaleSearchData = null) + internal static List Break(long localeId, List textStrings) { - return BreakCore(localeId, false, textStrings, LocaleSearchData); + return BreakCore(localeId, false, textStrings); } /// /// /// - internal static List Break(long localeId, string textString, LocaleWordBreakingData LocaleSearchData = null)// params string[] text) + internal static List Break(long localeId, string textString) { List textStrings = new List(1); textStrings.Add(textString); - return BreakCore(localeId, false, textStrings, LocaleSearchData); + return BreakCore(localeId, false, textStrings); } /// @@ -756,14 +753,21 @@ RETURNING id, xmin; /// Stop words list reset upon login or editing of localized text /// used for eliminating noise words from search dictionary /// - // public static System.Collections.Generic.List StopList = null; - internal static List BreakCore(long localeId, bool KeepWildCards, List textStrings, LocaleWordBreakingData LocaleSearchData = null) + public static Dictionary localeWordBreakingDataCache = new Dictionary(); + + internal static List BreakCore(long localeId, bool KeepWildCards, List textStrings) { //For stopwords and CJKIndex flag value //if not provided (will be provided by seeder for performance but normally never) then fetch - if (LocaleSearchData == null) - LocaleSearchData = GetLocaleSearchData(localeId); + + if (!localeWordBreakingDataCache.ContainsKey(localeId)) + { + localeWordBreakingDataCache.Add(localeId, GetLocaleSearchData(localeId)); + } + var localeWordBreakData = localeWordBreakingDataCache[localeId]; + + int MAXWORDLENGTH = 255; int MINWORDLENGTH = 2;//A word isn't a word unless it's got at least two characters in it StringBuilder sbResults = new StringBuilder(); @@ -797,7 +801,7 @@ RETURNING id, xmin; //get it as a character char c = t.GetTextElement()[0]; - if (!LocaleSearchData.CJKIndex) + if (!localeWordBreakData.CJKIndex) { #region regular tokenizer @@ -1027,7 +1031,7 @@ RETURNING id, xmin; if (s.Length >= MINWORDLENGTH || (KeepWildCards && s.Contains('%'))) { //Add only non stopwords - if (!LocaleSearchData.StopWords.Contains(s)) + if (!localeWordBreakData.StopWords.Contains(s)) { ReturnList.Add(s); } diff --git a/server/AyaNova/biz/UserBiz.cs b/server/AyaNova/biz/UserBiz.cs index 251c8a60..14291129 100644 --- a/server/AyaNova/biz/UserBiz.cs +++ b/server/AyaNova/biz/UserBiz.cs @@ -173,92 +173,92 @@ namespace AyaNova.Biz // } -// //get many (paged) -// internal async Task> GetManyAsync(IUrlHelper Url, string routeName, ListOptions pagingOptions) -// { + // //get many (paged) + // internal async Task> GetManyAsync(IUrlHelper Url, string routeName, ListOptions pagingOptions) + // { -// pagingOptions.Offset = pagingOptions.Offset ?? ListOptions.DefaultOffset; -// pagingOptions.Limit = pagingOptions.Limit ?? ListOptions.DefaultLimit; + // pagingOptions.Offset = pagingOptions.Offset ?? ListOptions.DefaultOffset; + // pagingOptions.Limit = pagingOptions.Limit ?? ListOptions.DefaultLimit; -// //BUILD THE QUERY -// //base query -// var q = "SELECT *, xmin FROM AUSER "; + // //BUILD THE QUERY + // //base query + // var q = "SELECT *, xmin FROM AUSER "; -// //GET THE FILTER / SORT -// if (pagingOptions.DataFilterId > 0) -// { -// var TheFilter = await ct.DataFilter.FirstOrDefaultAsync(x => x.Id == pagingOptions.DataFilterId); + // //GET THE FILTER / SORT + // if (pagingOptions.DataFilterId > 0) + // { + // var TheFilter = await ct.DataFilter.FirstOrDefaultAsync(x => x.Id == pagingOptions.DataFilterId); -// //BUILD WHERE AND APPEND IT -// q = q + SqlFilterCriteriaBuilder.DataFilterToSQLCriteria(TheFilter, AyaObjectFieldDefinitions.AyaObjectFields(AyaObjectFieldDefinitions.USER_KEY), UserId); + // //BUILD WHERE AND APPEND IT + // q = q + SqlFilterCriteriaBuilder.DataFilterToSQLCriteria(TheFilter, AyaObjectFieldDefinitions.AyaObjectFields(AyaObjectFieldDefinitions.USER_KEY), UserId); -// //BUILD ORDER BY AND APPEND IT -// q = q + SqlFilterOrderByBuilder.DataFilterToSQLOrderBy(TheFilter); -// } -// // else -// // { -// // //GET DEFAULT ORDER BY -// // q = q + SqlFilterOrderByBuilder.DefaultGetManyOrderBy; -// // } + // //BUILD ORDER BY AND APPEND IT + // q = q + SqlFilterOrderByBuilder.DataFilterToSQLOrderBy(TheFilter); + // } + // // else + // // { + // // //GET DEFAULT ORDER BY + // // q = q + SqlFilterOrderByBuilder.DefaultGetManyOrderBy; + // // } -// #pragma warning disable EF1000 + // #pragma warning disable EF1000 -// // var items = await ct.User -// // .AsNoTracking() -// // .FromSql(q) -// // .Skip(pagingOptions.Offset.Value) -// // .Take(pagingOptions.Limit.Value) -// // .ToArrayAsync(); + // // var items = await ct.User + // // .AsNoTracking() + // // .FromSql(q) + // // .Skip(pagingOptions.Offset.Value) + // // .Take(pagingOptions.Limit.Value) + // // .ToArrayAsync(); -// var items = await ct.User -// .FromSqlRaw(q) -// .AsNoTracking() -// .Skip(pagingOptions.Offset.Value) -// .Take(pagingOptions.Limit.Value) -// .ToArrayAsync(); + // var items = await ct.User + // .FromSqlRaw(q) + // .AsNoTracking() + // .Skip(pagingOptions.Offset.Value) + // .Take(pagingOptions.Limit.Value) + // .ToArrayAsync(); -// // var totalRecordCount = await ct.User -// // .AsNoTracking() -// // .FromSql(q) -// // .CountAsync(); + // // var totalRecordCount = await ct.User + // // .AsNoTracking() + // // .FromSql(q) + // // .CountAsync(); -// var totalRecordCount = await ct.User.FromSqlRaw(q) -// .AsNoTracking() -// .CountAsync(); -// #pragma warning restore EF1000 + // var totalRecordCount = await ct.User.FromSqlRaw(q) + // .AsNoTracking() + // .CountAsync(); + // #pragma warning restore EF1000 -// int itemCount = items.Count();//totalRecordCount doesn't skip and take so not usable here -// var cleanedItems = new System.Object[itemCount]; -// for (int i = 0; i < itemCount; i++) -// { -// cleanedItems[i] = CleanUserForReturn(items[i]); -// } + // int itemCount = items.Count();//totalRecordCount doesn't skip and take so not usable here + // var cleanedItems = new System.Object[itemCount]; + // for (int i = 0; i < itemCount; i++) + // { + // cleanedItems[i] = CleanUserForReturn(items[i]); + // } -// var pageLinks = new PaginationLinkBuilder(Url, routeName, null, pagingOptions, totalRecordCount).PagingLinksObject(); -// ApiPagedResponse pr = new ApiPagedResponse(cleanedItems, pageLinks); -// return pr; -// } + // var pageLinks = new PaginationLinkBuilder(Url, routeName, null, pagingOptions, totalRecordCount).PagingLinksObject(); + // ApiPagedResponse pr = new ApiPagedResponse(cleanedItems, pageLinks); + // return pr; + // } -// //get picklist (paged) -// internal ApiPagedResponse GetPickList(IUrlHelper Url, string routeName, ListOptions pagingOptions) -// { -// pagingOptions.Offset = pagingOptions.Offset ?? ListOptions.DefaultOffset; -// pagingOptions.Limit = pagingOptions.Limit ?? ListOptions.DefaultLimit; + // //get picklist (paged) + // internal ApiPagedResponse GetPickList(IUrlHelper Url, string routeName, ListOptions pagingOptions) + // { + // pagingOptions.Offset = pagingOptions.Offset ?? ListOptions.DefaultOffset; + // pagingOptions.Limit = pagingOptions.Limit ?? ListOptions.DefaultLimit; -// var ret = PickListFetcher.GetPickList(ct, UserId, pagingOptions, AyaObjectFieldDefinitions.AyaObjectFields(AyaObjectFieldDefinitions.USER_KEY), "auser"); + // var ret = PickListFetcher.GetPickList(ct, UserId, pagingOptions, AyaObjectFieldDefinitions.AyaObjectFields(AyaObjectFieldDefinitions.USER_KEY), "auser"); -// var pageLinks = new PaginationLinkBuilder(Url, routeName, null, pagingOptions, ret.TotalRecordCount).PagingLinksObject(); + // var pageLinks = new PaginationLinkBuilder(Url, routeName, null, pagingOptions, ret.TotalRecordCount).PagingLinksObject(); -// ApiPagedResponse pr = new ApiPagedResponse(ret.Items, pageLinks); -// return pr; -// } + // ApiPagedResponse pr = new ApiPagedResponse(ret.Items, pageLinks); + // return pr; + // } //////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/server/AyaNova/biz/WidgetBiz.cs b/server/AyaNova/biz/WidgetBiz.cs index e6110878..9ca38bc4 100644 --- a/server/AyaNova/biz/WidgetBiz.cs +++ b/server/AyaNova/biz/WidgetBiz.cs @@ -66,7 +66,7 @@ namespace AyaNova.Biz //CREATE //Called from route and also seeder - internal Widget Create(Widget inObj, Search.LocaleWordBreakingData LocaleSearchData = null) + internal Widget Create(Widget inObj) { Validate(inObj, null); if (HasErrors) @@ -93,7 +93,7 @@ namespace AyaNova.Biz //This takes 16 seconds out of 22 when seeding 500 widgets - SearchIndex(outObj, true, LocaleSearchData); + SearchIndex(outObj, true); //This takes 2 seconds out of 22 when seeding 500 widgets @@ -227,10 +227,10 @@ namespace AyaNova.Biz } - private void SearchIndex(Widget obj, bool isNew, Search.LocaleWordBreakingData LocaleSearchData=null) + private void SearchIndex(Widget obj, bool isNew) { //SEARCH INDEXING - var SearchParams = new Search.SearchIndexProcessObjectParameters(UserLocaleId, obj.Id, BizType, obj.Name, LocaleSearchData); + var SearchParams = new Search.SearchIndexProcessObjectParameters(UserLocaleId, obj.Id, BizType, obj.Name); SearchParams.AddText(obj.Notes).AddText(obj.Name).AddText(obj.Serial).AddText(obj.Tags).AddCustomFields(obj.CustomFields); diff --git a/server/AyaNova/util/Seeder.cs b/server/AyaNova/util/Seeder.cs index 449126d3..e883461e 100644 --- a/server/AyaNova/util/Seeder.cs +++ b/server/AyaNova/util/Seeder.cs @@ -17,15 +17,12 @@ namespace AyaNova.Util public enum SeedLevel { SmallOneManShopTrialDataSet, MediumLocalServiceCompanyTrialDataSet, LargeCorporateMultiRegionalTrialDataSet, HugeForLoadTest }; public static int SeededUserCount = 0; - //cache the locale word breaking data - private static Search.LocaleWordBreakingData LocaleSearchData = null; ////////////////////////////////////////////////////// //Seed database for trial and testing purposes // - public static void SeedDatabase(SeedLevel slevel, Decimal timeZoneOffset) { SeedDatabase(slevel, Guid.Empty, timeZoneOffset); @@ -74,13 +71,6 @@ namespace AyaNova.Util DbUtil.EmptyBizDataFromDatabaseForSeedingOrImporting(log); - - using (var cct = ServiceProviderProvider.DBContext) - { - //Get a cached LocaleWordBreakingData to speed up generation - LocaleSearchData = Search.GetLocaleSearchData(ServerBootConfig.AYANOVA_DEFAULT_LANGUAGE_ID, cct); - } - //Set the time zone of the manager account using (var cct = ServiceProviderProvider.DBContext) { @@ -630,7 +620,7 @@ namespace AyaNova.Util //var NewObject = Cached_WidgetBiz.CreateAsync(o).Result; //test without cached widgetbiz WidgetBiz biz = WidgetBiz.GetBizInternal(ServiceProviderProvider.DBContext); - var NewObject = biz.Create(o, LocaleSearchData); + var NewObject = biz.Create(o); if (NewObject == null) { log.LogError($"Seeder::GenSeedWidget error creating widget {o.Name}\r\n" + biz.GetErrorsAsString());