admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Browse Source
This commit is contained in:
John Cardinal
2020-01-24 20:35:28 +00:00
parent b241561fe6
commit 008ea40649
5 changed files with 96 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
devdocs/todo.txt
View File

@@ -48,12 +48,18 @@ Async the keyword processing
Removing use of EF entirely in search indexing processing in favor of direct sql queries Removing use of EF entirely in search indexing processing in favor of direct sql queries
cache or provide directly the locale to save time repeatedly fetching it when doing bulk ops cache or provide directly the locale to save time repeatedly fetching it when doing bulk ops!!!
-After doing this 500=21 That's as fast as when I stripped out all the text, what a huge overhead saving right there!:
2020-01-24 12:00:41.2547|INFO|Seeder|Seeding 500 Widgets....
2020-01-24 12:00:51.9138|INFO|Seeder|500 Widgets seeded in 10649 ms
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
TODO: Locale search data, there is overhead on every record save due to having to repeatedly fetch this
- While I added a way to seed it without fetching, there should actually just be a cached version for all record saving stored locally
- Fetches in the db are not necessary for this object more than once or on change of locale
- So probably Search code should cache it internally on first fetch
TODO: RUN data list query with debug info on and see if any EF stuff is creeping into it TODO: RUN data list query with debug info on and see if any EF stuff is creeping into it

36
server/AyaNova/biz/Search.cs
View File

@@ -19,7 +19,6 @@ namespace AyaNova.Biz
//This class handles word breaking, processing keywords and searching for results //This class handles word breaking, processing keywords and searching for results
public static class Search public static class Search
{ {
//private static ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("Search");
#region Search and return results #region Search and return results
@@ -379,20 +378,18 @@ namespace AyaNova.Biz
public long ObjectId { get; set; } public long ObjectId { get; set; }
public AyaType ObjectType { get; set; } public AyaType ObjectType { get; set; }
public string Name { get; set; } public string Name { get; set; }
public List<string> Words { get; set; } public List<string> Words { get; set; }
public LocaleWordBreakingData LocaleSearchData { get; set; }
public SearchIndexProcessObjectParameters(long localeId, long objectID, AyaType objectType, string name, LocaleWordBreakingData localeSearchData = null) public SearchIndexProcessObjectParameters(long localeId, long objectID, AyaType objectType, string name)
{ {
Words = new List<string>(); Words = new List<string>();
LocaleId = localeId; LocaleId = localeId;
ObjectId = objectID; ObjectId = objectID;
ObjectType = objectType; ObjectType = objectType;
Name = name; Name = name;
LocaleSearchData = localeSearchData;
} }
@@ -726,19 +723,19 @@ RETURNING id, xmin;
/// (languages with no easily identifiable word boundaries as in english) /// (languages with no easily identifiable word boundaries as in english)
/// </summary> /// </summary>
/// <returns>List of strings</returns> /// <returns>List of strings</returns>
internal static List<string> Break(long localeId, List<string> textStrings, LocaleWordBreakingData LocaleSearchData = null) internal static List<string> Break(long localeId, List<string> textStrings)
{ {
return BreakCore(localeId, false, textStrings, LocaleSearchData); return BreakCore(localeId, false, textStrings);
} }
/// <summary> /// <summary>
/// ///
/// </summary> /// </summary>
internal static List<string> Break(long localeId, string textString, LocaleWordBreakingData LocaleSearchData = null)// params string[] text) internal static List<string> Break(long localeId, string textString)
{ {
List<string> textStrings = new List<string>(1); List<string> textStrings = new List<string>(1);
textStrings.Add(textString); textStrings.Add(textString);
return BreakCore(localeId, false, textStrings, LocaleSearchData); return BreakCore(localeId, false, textStrings);
} }
/// <summary> /// <summary>
@@ -756,14 +753,21 @@ RETURNING id, xmin;
/// Stop words list reset upon login or editing of localized text /// Stop words list reset upon login or editing of localized text
/// used for eliminating noise words from search dictionary /// used for eliminating noise words from search dictionary
/// </summary> /// </summary>
// public static System.Collections.Generic.List<string> StopList = null;
internal static List<string> BreakCore(long localeId, bool KeepWildCards, List<string> textStrings, LocaleWordBreakingData LocaleSearchData = null) public static Dictionary<long, LocaleWordBreakingData> localeWordBreakingDataCache = new Dictionary<long, LocaleWordBreakingData>();
internal static List<string> BreakCore(long localeId, bool KeepWildCards, List<string> textStrings)
{ {
//For stopwords and CJKIndex flag value //For stopwords and CJKIndex flag value
//if not provided (will be provided by seeder for performance but normally never) then fetch //if not provided (will be provided by seeder for performance but normally never) then fetch
if (LocaleSearchData == null)
LocaleSearchData = GetLocaleSearchData(localeId); if (!localeWordBreakingDataCache.ContainsKey(localeId))
{
localeWordBreakingDataCache.Add(localeId, GetLocaleSearchData(localeId));
}
var localeWordBreakData = localeWordBreakingDataCache[localeId];
int MAXWORDLENGTH = 255; int MAXWORDLENGTH = 255;
int MINWORDLENGTH = 2;//A word isn't a word unless it's got at least two characters in it int MINWORDLENGTH = 2;//A word isn't a word unless it's got at least two characters in it
StringBuilder sbResults = new StringBuilder(); StringBuilder sbResults = new StringBuilder();
@@ -797,7 +801,7 @@ RETURNING id, xmin;
//get it as a character //get it as a character
char c = t.GetTextElement()[0]; char c = t.GetTextElement()[0];
if (!LocaleSearchData.CJKIndex) if (!localeWordBreakData.CJKIndex)
{ {
#region regular tokenizer #region regular tokenizer
@@ -1027,7 +1031,7 @@ RETURNING id, xmin;
if (s.Length >= MINWORDLENGTH || (KeepWildCards && s.Contains('%'))) if (s.Length >= MINWORDLENGTH || (KeepWildCards && s.Contains('%')))
{ {
//Add only non stopwords //Add only non stopwords
if (!LocaleSearchData.StopWords.Contains(s)) if (!localeWordBreakData.StopWords.Contains(s))
{ {
ReturnList.Add(s); ReturnList.Add(s);
} }

126
server/AyaNova/biz/UserBiz.cs
View File

@@ -173,92 +173,92 @@ namespace AyaNova.Biz
// } // }
// //get many (paged) // //get many (paged)
// internal async Task<ApiPagedResponse<System.Object>> GetManyAsync(IUrlHelper Url, string routeName, ListOptions pagingOptions) // internal async Task<ApiPagedResponse<System.Object>> GetManyAsync(IUrlHelper Url, string routeName, ListOptions pagingOptions)
// { // {
// pagingOptions.Offset = pagingOptions.Offset ?? ListOptions.DefaultOffset; // pagingOptions.Offset = pagingOptions.Offset ?? ListOptions.DefaultOffset;
// pagingOptions.Limit = pagingOptions.Limit ?? ListOptions.DefaultLimit; // pagingOptions.Limit = pagingOptions.Limit ?? ListOptions.DefaultLimit;
// //BUILD THE QUERY // //BUILD THE QUERY
// //base query // //base query
// var q = "SELECT *, xmin FROM AUSER "; // var q = "SELECT *, xmin FROM AUSER ";
// //GET THE FILTER / SORT // //GET THE FILTER / SORT
// if (pagingOptions.DataFilterId > 0) // if (pagingOptions.DataFilterId > 0)
// { // {
// var TheFilter = await ct.DataFilter.FirstOrDefaultAsync(x => x.Id == pagingOptions.DataFilterId); // var TheFilter = await ct.DataFilter.FirstOrDefaultAsync(x => x.Id == pagingOptions.DataFilterId);
// //BUILD WHERE AND APPEND IT // //BUILD WHERE AND APPEND IT
// q = q + SqlFilterCriteriaBuilder.DataFilterToSQLCriteria(TheFilter, AyaObjectFieldDefinitions.AyaObjectFields(AyaObjectFieldDefinitions.USER_KEY), UserId); // q = q + SqlFilterCriteriaBuilder.DataFilterToSQLCriteria(TheFilter, AyaObjectFieldDefinitions.AyaObjectFields(AyaObjectFieldDefinitions.USER_KEY), UserId);
// //BUILD ORDER BY AND APPEND IT // //BUILD ORDER BY AND APPEND IT
// q = q + SqlFilterOrderByBuilder.DataFilterToSQLOrderBy(TheFilter); // q = q + SqlFilterOrderByBuilder.DataFilterToSQLOrderBy(TheFilter);
// } // }
// // else // // else
// // { // // {
// // //GET DEFAULT ORDER BY // // //GET DEFAULT ORDER BY
// // q = q + SqlFilterOrderByBuilder.DefaultGetManyOrderBy; // // q = q + SqlFilterOrderByBuilder.DefaultGetManyOrderBy;
// // } // // }
// #pragma warning disable EF1000 // #pragma warning disable EF1000
// // var items = await ct.User // // var items = await ct.User
// // .AsNoTracking() // // .AsNoTracking()
// // .FromSql(q) // // .FromSql(q)
// // .Skip(pagingOptions.Offset.Value) // // .Skip(pagingOptions.Offset.Value)
// // .Take(pagingOptions.Limit.Value) // // .Take(pagingOptions.Limit.Value)
// // .ToArrayAsync(); // // .ToArrayAsync();
// var items = await ct.User // var items = await ct.User
// .FromSqlRaw(q) // .FromSqlRaw(q)
// .AsNoTracking() // .AsNoTracking()
// .Skip(pagingOptions.Offset.Value) // .Skip(pagingOptions.Offset.Value)
// .Take(pagingOptions.Limit.Value) // .Take(pagingOptions.Limit.Value)
// .ToArrayAsync(); // .ToArrayAsync();
// // var totalRecordCount = await ct.User // // var totalRecordCount = await ct.User
// // .AsNoTracking() // // .AsNoTracking()
// // .FromSql(q) // // .FromSql(q)
// // .CountAsync(); // // .CountAsync();
// var totalRecordCount = await ct.User.FromSqlRaw(q) // var totalRecordCount = await ct.User.FromSqlRaw(q)
// .AsNoTracking() // .AsNoTracking()
// .CountAsync(); // .CountAsync();
// #pragma warning restore EF1000 // #pragma warning restore EF1000
// int itemCount = items.Count();//totalRecordCount doesn't skip and take so not usable here // int itemCount = items.Count();//totalRecordCount doesn't skip and take so not usable here
// var cleanedItems = new System.Object[itemCount]; // var cleanedItems = new System.Object[itemCount];
// for (int i = 0; i < itemCount; i++) // for (int i = 0; i < itemCount; i++)
// { // {
// cleanedItems[i] = CleanUserForReturn(items[i]); // cleanedItems[i] = CleanUserForReturn(items[i]);
// } // }
// var pageLinks = new PaginationLinkBuilder(Url, routeName, null, pagingOptions, totalRecordCount).PagingLinksObject(); // var pageLinks = new PaginationLinkBuilder(Url, routeName, null, pagingOptions, totalRecordCount).PagingLinksObject();
// ApiPagedResponse<System.Object> pr = new ApiPagedResponse<System.Object>(cleanedItems, pageLinks); // ApiPagedResponse<System.Object> pr = new ApiPagedResponse<System.Object>(cleanedItems, pageLinks);
// return pr; // return pr;
// } // }
// //get picklist (paged) // //get picklist (paged)
// internal ApiPagedResponse<NameIdItem> GetPickList(IUrlHelper Url, string routeName, ListOptions pagingOptions) // internal ApiPagedResponse<NameIdItem> GetPickList(IUrlHelper Url, string routeName, ListOptions pagingOptions)
// { // {
// pagingOptions.Offset = pagingOptions.Offset ?? ListOptions.DefaultOffset; // pagingOptions.Offset = pagingOptions.Offset ?? ListOptions.DefaultOffset;
// pagingOptions.Limit = pagingOptions.Limit ?? ListOptions.DefaultLimit; // pagingOptions.Limit = pagingOptions.Limit ?? ListOptions.DefaultLimit;
// var ret = PickListFetcher.GetPickList(ct, UserId, pagingOptions, AyaObjectFieldDefinitions.AyaObjectFields(AyaObjectFieldDefinitions.USER_KEY), "auser"); // var ret = PickListFetcher.GetPickList(ct, UserId, pagingOptions, AyaObjectFieldDefinitions.AyaObjectFields(AyaObjectFieldDefinitions.USER_KEY), "auser");
// var pageLinks = new PaginationLinkBuilder(Url, routeName, null, pagingOptions, ret.TotalRecordCount).PagingLinksObject(); // var pageLinks = new PaginationLinkBuilder(Url, routeName, null, pagingOptions, ret.TotalRecordCount).PagingLinksObject();
// ApiPagedResponse<NameIdItem> pr = new ApiPagedResponse<NameIdItem>(ret.Items, pageLinks); // ApiPagedResponse<NameIdItem> pr = new ApiPagedResponse<NameIdItem>(ret.Items, pageLinks);
// return pr; // return pr;
// } // }
//////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////

8
server/AyaNova/biz/WidgetBiz.cs
View File

@@ -66,7 +66,7 @@ namespace AyaNova.Biz
//CREATE //CREATE
//Called from route and also seeder //Called from route and also seeder
internal Widget Create(Widget inObj, Search.LocaleWordBreakingData LocaleSearchData = null) internal Widget Create(Widget inObj)
{ {
Validate(inObj, null); Validate(inObj, null);
if (HasErrors) if (HasErrors)
@@ -93,7 +93,7 @@ namespace AyaNova.Biz
//This takes 16 seconds out of 22 when seeding 500 widgets //This takes 16 seconds out of 22 when seeding 500 widgets
SearchIndex(outObj, true, LocaleSearchData); SearchIndex(outObj, true);
//This takes 2 seconds out of 22 when seeding 500 widgets //This takes 2 seconds out of 22 when seeding 500 widgets
@@ -227,10 +227,10 @@ namespace AyaNova.Biz
} }
private void SearchIndex(Widget obj, bool isNew, Search.LocaleWordBreakingData LocaleSearchData=null) private void SearchIndex(Widget obj, bool isNew)
{ {
//SEARCH INDEXING //SEARCH INDEXING
var SearchParams = new Search.SearchIndexProcessObjectParameters(UserLocaleId, obj.Id, BizType, obj.Name, LocaleSearchData); var SearchParams = new Search.SearchIndexProcessObjectParameters(UserLocaleId, obj.Id, BizType, obj.Name);
SearchParams.AddText(obj.Notes).AddText(obj.Name).AddText(obj.Serial).AddText(obj.Tags).AddCustomFields(obj.CustomFields); SearchParams.AddText(obj.Notes).AddText(obj.Name).AddText(obj.Serial).AddText(obj.Tags).AddCustomFields(obj.CustomFields);

12
server/AyaNova/util/Seeder.cs
View File

@@ -17,15 +17,12 @@ namespace AyaNova.Util
public enum SeedLevel { SmallOneManShopTrialDataSet, MediumLocalServiceCompanyTrialDataSet, LargeCorporateMultiRegionalTrialDataSet, HugeForLoadTest }; public enum SeedLevel { SmallOneManShopTrialDataSet, MediumLocalServiceCompanyTrialDataSet, LargeCorporateMultiRegionalTrialDataSet, HugeForLoadTest };
public static int SeededUserCount = 0; public static int SeededUserCount = 0;
//cache the locale word breaking data
private static Search.LocaleWordBreakingData LocaleSearchData = null;
////////////////////////////////////////////////////// //////////////////////////////////////////////////////
//Seed database for trial and testing purposes //Seed database for trial and testing purposes
// //
public static void SeedDatabase(SeedLevel slevel, Decimal timeZoneOffset) public static void SeedDatabase(SeedLevel slevel, Decimal timeZoneOffset)
{ {
SeedDatabase(slevel, Guid.Empty, timeZoneOffset); SeedDatabase(slevel, Guid.Empty, timeZoneOffset);
@@ -74,13 +71,6 @@ namespace AyaNova.Util
DbUtil.EmptyBizDataFromDatabaseForSeedingOrImporting(log); DbUtil.EmptyBizDataFromDatabaseForSeedingOrImporting(log);
using (var cct = ServiceProviderProvider.DBContext)
{
//Get a cached LocaleWordBreakingData to speed up generation
LocaleSearchData = Search.GetLocaleSearchData(ServerBootConfig.AYANOVA_DEFAULT_LANGUAGE_ID, cct);
}
//Set the time zone of the manager account //Set the time zone of the manager account
using (var cct = ServiceProviderProvider.DBContext) using (var cct = ServiceProviderProvider.DBContext)
{ {
@@ -630,7 +620,7 @@ namespace AyaNova.Util
//var NewObject = Cached_WidgetBiz.CreateAsync(o).Result; //var NewObject = Cached_WidgetBiz.CreateAsync(o).Result;
//test without cached widgetbiz //test without cached widgetbiz
WidgetBiz biz = WidgetBiz.GetBizInternal(ServiceProviderProvider.DBContext); WidgetBiz biz = WidgetBiz.GetBizInternal(ServiceProviderProvider.DBContext);
var NewObject = biz.Create(o, LocaleSearchData); var NewObject = biz.Create(o);
if (NewObject == null) if (NewObject == null)
{ {
log.LogError($"Seeder::GenSeedWidget error creating widget {o.Name}\r\n" + biz.GetErrorsAsString()); log.LogError($"Seeder::GenSeedWidget error creating widget {o.Name}\r\n" + biz.GetErrorsAsString());