admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Browse Source
This commit is contained in:
John Cardinal
2020-01-24 20:35:28 +00:00
parent b241561fe6
commit 008ea40649
5 changed files with 96 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
devdocs/todo.txt
View File

@@ -48,12 +48,18 @@ Async the keyword processing
Removing use of EF entirely in search indexing processing in favor of direct sql queries
cache or provide directly the locale to save time repeatedly fetching it when doing bulk ops
cache or provide directly the locale to save time repeatedly fetching it when doing bulk ops!!!
-After doing this 500=21 That's as fast as when I stripped out all the text, what a huge overhead saving right there!:
2020-01-24 12:00:41.2547|INFO|Seeder|Seeding 500 Widgets....
2020-01-24 12:00:51.9138|INFO|Seeder|500 Widgets seeded in 10649 ms
///////////////////////////////////////////////////////////////////////////////
TODO: Locale search data, there is overhead on every record save due to having to repeatedly fetch this
- While I added a way to seed it without fetching, there should actually just be a cached version for all record saving stored locally
- Fetches in the db are not necessary for this object more than once or on change of locale
- So probably Search code should cache it internally on first fetch
TODO: RUN data list query with debug info on and see if any EF stuff is creeping into it

36
server/AyaNova/biz/Search.cs
View File

@@ -19,7 +19,6 @@ namespace AyaNova.Biz
//This class handles word breaking, processing keywords and searching for results
public static class Search
{
//private static ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("Search");
#region Search and return results
@@ -379,20 +378,18 @@ namespace AyaNova.Biz
public long ObjectId { get; set; }
public AyaType ObjectType { get; set; }
public string Name { get; set; }
public List<string> Words { get; set; }
public LocaleWordBreakingData LocaleSearchData { get; set; }
public List<string> Words { get; set; }
public SearchIndexProcessObjectParameters(long localeId, long objectID, AyaType objectType, string name, LocaleWordBreakingData localeSearchData = null)
public SearchIndexProcessObjectParameters(long localeId, long objectID, AyaType objectType, string name)
{
Words = new List<string>();
LocaleId = localeId;
ObjectId = objectID;
ObjectType = objectType;
Name = name;
LocaleSearchData = localeSearchData;
Name = name;
}
@@ -726,19 +723,19 @@ RETURNING id, xmin;
/// (languages with no easily identifiable word boundaries as in english)
/// </summary>
/// <returns>List of strings</returns>
internal static List<string> Break(long localeId, List<string> textStrings, LocaleWordBreakingData LocaleSearchData = null)
internal static List<string> Break(long localeId, List<string> textStrings)
{
return BreakCore(localeId, false, textStrings, LocaleSearchData);
return BreakCore(localeId, false, textStrings);
}
/// <summary>
///
/// </summary>
internal static List<string> Break(long localeId, string textString, LocaleWordBreakingData LocaleSearchData = null)// params string[] text)
internal static List<string> Break(long localeId, string textString)
{
List<string> textStrings = new List<string>(1);
textStrings.Add(textString);
return BreakCore(localeId, false, textStrings, LocaleSearchData);
return BreakCore(localeId, false, textStrings);
}
/// <summary>
@@ -756,14 +753,21 @@ RETURNING id, xmin;
/// Stop words list reset upon login or editing of localized text
/// used for eliminating noise words from search dictionary
/// </summary>
// public static System.Collections.Generic.List<string> StopList = null;
internal static List<string> BreakCore(long localeId, bool KeepWildCards, List<string> textStrings, LocaleWordBreakingData LocaleSearchData = null)
public static Dictionary<long, LocaleWordBreakingData> localeWordBreakingDataCache = new Dictionary<long, LocaleWordBreakingData>();
internal static List<string> BreakCore(long localeId, bool KeepWildCards, List<string> textStrings)
{
//For stopwords and CJKIndex flag value
//if not provided (will be provided by seeder for performance but normally never) then fetch
if (LocaleSearchData == null)
LocaleSearchData = GetLocaleSearchData(localeId);
if (!localeWordBreakingDataCache.ContainsKey(localeId))
{
localeWordBreakingDataCache.Add(localeId, GetLocaleSearchData(localeId));
}
var localeWordBreakData = localeWordBreakingDataCache[localeId];
int MAXWORDLENGTH = 255;
int MINWORDLENGTH = 2;//A word isn't a word unless it's got at least two characters in it
StringBuilder sbResults = new StringBuilder();
@@ -797,7 +801,7 @@ RETURNING id, xmin;
//get it as a character
char c = t.GetTextElement()[0];
if (!LocaleSearchData.CJKIndex)
if (!localeWordBreakData.CJKIndex)
{
#region regular tokenizer
@@ -1027,7 +1031,7 @@ RETURNING id, xmin;
if (s.Length >= MINWORDLENGTH || (KeepWildCards && s.Contains('%')))
{
//Add only non stopwords
if (!LocaleSearchData.StopWords.Contains(s))
if (!localeWordBreakData.StopWords.Contains(s))
{
ReturnList.Add(s);
}

126
server/AyaNova/biz/UserBiz.cs
View File

@@ -173,92 +173,92 @@ namespace AyaNova.Biz
// }
// //get many (paged)
// internal async Task<ApiPagedResponse<System.Object>> GetManyAsync(IUrlHelper Url, string routeName, ListOptions pagingOptions)
// {
// //get many (paged)
// internal async Task<ApiPagedResponse<System.Object>> GetManyAsync(IUrlHelper Url, string routeName, ListOptions pagingOptions)
// {
// pagingOptions.Offset = pagingOptions.Offset ?? ListOptions.DefaultOffset;
// pagingOptions.Limit = pagingOptions.Limit ?? ListOptions.DefaultLimit;
// pagingOptions.Offset = pagingOptions.Offset ?? ListOptions.DefaultOffset;
// pagingOptions.Limit = pagingOptions.Limit ?? ListOptions.DefaultLimit;
// //BUILD THE QUERY
// //base query
// var q = "SELECT *, xmin FROM AUSER ";
// //BUILD THE QUERY
// //base query
// var q = "SELECT *, xmin FROM AUSER ";
// //GET THE FILTER / SORT
// if (pagingOptions.DataFilterId > 0)
// {
// var TheFilter = await ct.DataFilter.FirstOrDefaultAsync(x => x.Id == pagingOptions.DataFilterId);
// //GET THE FILTER / SORT
// if (pagingOptions.DataFilterId > 0)
// {
// var TheFilter = await ct.DataFilter.FirstOrDefaultAsync(x => x.Id == pagingOptions.DataFilterId);
// //BUILD WHERE AND APPEND IT
// q = q + SqlFilterCriteriaBuilder.DataFilterToSQLCriteria(TheFilter, AyaObjectFieldDefinitions.AyaObjectFields(AyaObjectFieldDefinitions.USER_KEY), UserId);
// //BUILD WHERE AND APPEND IT
// q = q + SqlFilterCriteriaBuilder.DataFilterToSQLCriteria(TheFilter, AyaObjectFieldDefinitions.AyaObjectFields(AyaObjectFieldDefinitions.USER_KEY), UserId);
// //BUILD ORDER BY AND APPEND IT
// q = q + SqlFilterOrderByBuilder.DataFilterToSQLOrderBy(TheFilter);
// }
// // else
// // {
// // //GET DEFAULT ORDER BY
// // q = q + SqlFilterOrderByBuilder.DefaultGetManyOrderBy;
// // }
// //BUILD ORDER BY AND APPEND IT
// q = q + SqlFilterOrderByBuilder.DataFilterToSQLOrderBy(TheFilter);
// }
// // else
// // {
// // //GET DEFAULT ORDER BY
// // q = q + SqlFilterOrderByBuilder.DefaultGetManyOrderBy;
// // }
// #pragma warning disable EF1000
// #pragma warning disable EF1000
// // var items = await ct.User
// // .AsNoTracking()
// // .FromSql(q)
// // .Skip(pagingOptions.Offset.Value)
// // .Take(pagingOptions.Limit.Value)
// // .ToArrayAsync();
// // var items = await ct.User
// // .AsNoTracking()
// // .FromSql(q)
// // .Skip(pagingOptions.Offset.Value)
// // .Take(pagingOptions.Limit.Value)
// // .ToArrayAsync();
// var items = await ct.User
// .FromSqlRaw(q)
// .AsNoTracking()
// .Skip(pagingOptions.Offset.Value)
// .Take(pagingOptions.Limit.Value)
// .ToArrayAsync();
// var items = await ct.User
// .FromSqlRaw(q)
// .AsNoTracking()
// .Skip(pagingOptions.Offset.Value)
// .Take(pagingOptions.Limit.Value)
// .ToArrayAsync();
// // var totalRecordCount = await ct.User
// // .AsNoTracking()
// // .FromSql(q)
// // .CountAsync();
// // var totalRecordCount = await ct.User
// // .AsNoTracking()
// // .FromSql(q)
// // .CountAsync();
// var totalRecordCount = await ct.User.FromSqlRaw(q)
// .AsNoTracking()
// .CountAsync();
// #pragma warning restore EF1000
// var totalRecordCount = await ct.User.FromSqlRaw(q)
// .AsNoTracking()
// .CountAsync();
// #pragma warning restore EF1000
// int itemCount = items.Count();//totalRecordCount doesn't skip and take so not usable here
// var cleanedItems = new System.Object[itemCount];
// for (int i = 0; i < itemCount; i++)
// {
// cleanedItems[i] = CleanUserForReturn(items[i]);
// }
// int itemCount = items.Count();//totalRecordCount doesn't skip and take so not usable here
// var cleanedItems = new System.Object[itemCount];
// for (int i = 0; i < itemCount; i++)
// {
// cleanedItems[i] = CleanUserForReturn(items[i]);
// }
// var pageLinks = new PaginationLinkBuilder(Url, routeName, null, pagingOptions, totalRecordCount).PagingLinksObject();
// ApiPagedResponse<System.Object> pr = new ApiPagedResponse<System.Object>(cleanedItems, pageLinks);
// return pr;
// }
// var pageLinks = new PaginationLinkBuilder(Url, routeName, null, pagingOptions, totalRecordCount).PagingLinksObject();
// ApiPagedResponse<System.Object> pr = new ApiPagedResponse<System.Object>(cleanedItems, pageLinks);
// return pr;
// }
// //get picklist (paged)
// internal ApiPagedResponse<NameIdItem> GetPickList(IUrlHelper Url, string routeName, ListOptions pagingOptions)
// {
// pagingOptions.Offset = pagingOptions.Offset ?? ListOptions.DefaultOffset;
// pagingOptions.Limit = pagingOptions.Limit ?? ListOptions.DefaultLimit;
// //get picklist (paged)
// internal ApiPagedResponse<NameIdItem> GetPickList(IUrlHelper Url, string routeName, ListOptions pagingOptions)
// {
// pagingOptions.Offset = pagingOptions.Offset ?? ListOptions.DefaultOffset;
// pagingOptions.Limit = pagingOptions.Limit ?? ListOptions.DefaultLimit;
// var ret = PickListFetcher.GetPickList(ct, UserId, pagingOptions, AyaObjectFieldDefinitions.AyaObjectFields(AyaObjectFieldDefinitions.USER_KEY), "auser");
// var ret = PickListFetcher.GetPickList(ct, UserId, pagingOptions, AyaObjectFieldDefinitions.AyaObjectFields(AyaObjectFieldDefinitions.USER_KEY), "auser");
// var pageLinks = new PaginationLinkBuilder(Url, routeName, null, pagingOptions, ret.TotalRecordCount).PagingLinksObject();
// var pageLinks = new PaginationLinkBuilder(Url, routeName, null, pagingOptions, ret.TotalRecordCount).PagingLinksObject();
// ApiPagedResponse<NameIdItem> pr = new ApiPagedResponse<NameIdItem>(ret.Items, pageLinks);
// return pr;
// }
// ApiPagedResponse<NameIdItem> pr = new ApiPagedResponse<NameIdItem>(ret.Items, pageLinks);
// return pr;
// }
////////////////////////////////////////////////////////////////////////////////////////////////

8
server/AyaNova/biz/WidgetBiz.cs
View File

@@ -66,7 +66,7 @@ namespace AyaNova.Biz
//CREATE
//Called from route and also seeder
internal Widget Create(Widget inObj, Search.LocaleWordBreakingData LocaleSearchData = null)
internal Widget Create(Widget inObj)
{
Validate(inObj, null);
if (HasErrors)
@@ -93,7 +93,7 @@ namespace AyaNova.Biz
//This takes 16 seconds out of 22 when seeding 500 widgets
SearchIndex(outObj, true, LocaleSearchData);
SearchIndex(outObj, true);
//This takes 2 seconds out of 22 when seeding 500 widgets
@@ -227,10 +227,10 @@ namespace AyaNova.Biz
}
private void SearchIndex(Widget obj, bool isNew, Search.LocaleWordBreakingData LocaleSearchData=null)
private void SearchIndex(Widget obj, bool isNew)
{
//SEARCH INDEXING
var SearchParams = new Search.SearchIndexProcessObjectParameters(UserLocaleId, obj.Id, BizType, obj.Name, LocaleSearchData);
var SearchParams = new Search.SearchIndexProcessObjectParameters(UserLocaleId, obj.Id, BizType, obj.Name);
SearchParams.AddText(obj.Notes).AddText(obj.Name).AddText(obj.Serial).AddText(obj.Tags).AddCustomFields(obj.CustomFields);

12
server/AyaNova/util/Seeder.cs
View File

@@ -17,15 +17,12 @@ namespace AyaNova.Util
public enum SeedLevel { SmallOneManShopTrialDataSet, MediumLocalServiceCompanyTrialDataSet, LargeCorporateMultiRegionalTrialDataSet, HugeForLoadTest };
public static int SeededUserCount = 0;
//cache the locale word breaking data
private static Search.LocaleWordBreakingData LocaleSearchData = null;
//////////////////////////////////////////////////////
//Seed database for trial and testing purposes
//
public static void SeedDatabase(SeedLevel slevel, Decimal timeZoneOffset)
{
SeedDatabase(slevel, Guid.Empty, timeZoneOffset);
@@ -74,13 +71,6 @@ namespace AyaNova.Util
DbUtil.EmptyBizDataFromDatabaseForSeedingOrImporting(log);
using (var cct = ServiceProviderProvider.DBContext)
{
//Get a cached LocaleWordBreakingData to speed up generation
LocaleSearchData = Search.GetLocaleSearchData(ServerBootConfig.AYANOVA_DEFAULT_LANGUAGE_ID, cct);
}
//Set the time zone of the manager account
using (var cct = ServiceProviderProvider.DBContext)
{
@@ -630,7 +620,7 @@ namespace AyaNova.Util
//var NewObject = Cached_WidgetBiz.CreateAsync(o).Result;
//test without cached widgetbiz
WidgetBiz biz = WidgetBiz.GetBizInternal(ServiceProviderProvider.DBContext);
var NewObject = biz.Create(o, LocaleSearchData);
var NewObject = biz.Create(o);
if (NewObject == null)
{
log.LogError($"Seeder::GenSeedWidget error creating widget {o.Name}\r\n" + biz.GetErrorsAsString());