admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Browse Source
This commit is contained in:
John Cardinal
2020-01-24 19:55:26 +00:00
parent 183e52202c
commit b241561fe6
6 changed files with 69 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.vscode/launch.json vendored
View File

@@ -40,8 +40,8 @@
"env": {
"ASPNETCORE_ENVIRONMENT": "Development",
"AYANOVA_JWT_SECRET": "UNLICENSED5G*QQJ8#bQ7$Xr_@sXfHq4",
//"AYANOVA_LOG_LEVEL": "Info",
"AYANOVA_LOG_LEVEL": "Debug",
"AYANOVA_LOG_LEVEL": "Info",
//"AYANOVA_LOG_LEVEL": "Debug",
"AYANOVA_DEFAULT_LANGUAGE": "en",
//LOCALE MUST BE en for Integration TESTING
//"AYANOVA_PERMANENTLY_ERASE_DATABASE": "true",

30
devdocs/todo.txt
View File

@@ -5,7 +5,13 @@ eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOiIxNTcxODU5OTU0IiwiZXhwIjoiMTU3MjQ
## IMMEDIATE ITEMS
PERFORMANCE WORK
///////////////////////////////////////////////////////////////////////////////
TODO: Search indexing is painfully slow, it accounts for 16 of 22 seconds when creating 500 widgets with full paragraphs of text
- Try to see if it's just one part of the operation by timing it
- Re-code it not using EF but directly interacting with the DB
- Maybe it's a case for stored procedures or something?
SEARCH INDEXING PERFORMANCE WORK
Baseline from before doing anything seeding a medium level with full text
2020-01-21 16:49:17.4662|INFO|Seeder|75 Users seeded in 2279 ms
2020-01-21 16:49:39.4481|INFO|Seeder|500 Widgets seeded in 21968 ms
@@ -30,16 +36,26 @@ Now going to try the opposite, a *lot* of text 10 paragraphs in both c2 and note
So the quantity of text directly affects the performance, so it's not just some overhead from the query being run, it's the amount of work it needs to do in the queries
THINGS TO TRY:
Completely alternate methods:
- https://stackoverflow.com/a/15089664/8939 Store a Digest of each record with that record then can just search the digests (would mean a search has to traverse all records of every table possibly)
DB INDEX TUNING?
- Play with the indexes and see if there is a slowup with an unnecessary index maybe affecting things
Async the keyword processing
- Fire off the indexing and return immediately so there would be a bit of time to come into compliance maybe more clashes?
Removing use of EF entirely in search indexing processing in favor of direct sql queries
cache or provide directly the locale to save time repeatedly fetching it when doing bulk ops
///////////////////////////////////////////////////////////////////////////////
TODO: Search indexing is painfully slow, it accounts for 16 of 22 seconds when creating 500 widgets with full paragraphs of text
- Try to see if it's just one part of the operation by timing it
- Re-code it not using EF but directly interacting with the DB
- Maybe it's a case for stored procedures or something?
TODO: RUN data list query with debug info on and see if any EF stuff is creeping into it
TODO: DataFilter how to distinguish between filtering on specific ID value or on value column

4
server/AyaNova/Startup.cs
View File

@@ -134,7 +134,7 @@ namespace AyaNova
bool LOG_SENSITIVE_DATA = false;
#if (DEBUG)
LOG_SENSITIVE_DATA = true;//############################################################################
LOG_SENSITIVE_DATA = false;//############################################################################
#endif
@@ -449,7 +449,7 @@ namespace AyaNova
{
AyaNova.Core.License.Fetch(apiServerState, dbContext, _newLog);
//NOTE: For unit testing make sure the time zone in util is set to the same figure as here to ensure list filter by date tests will work because server is on same page as user in terms of time
Util.Seeder.SeedDatabase(Util.Seeder.SeedLevel.SmallOneManShopTrialDataSet, -7);//#############################################################################################
Util.Seeder.SeedDatabase(Util.Seeder.SeedLevel.MediumLocalServiceCompanyTrialDataSet, -7);//#############################################################################################
}
//TESTING
#endif

32
server/AyaNova/biz/Search.cs
View File

@@ -380,10 +380,11 @@ namespace AyaNova.Biz
public AyaType ObjectType { get; set; }
public string Name { get; set; }
public List<string> Words { get; set; }
public LocaleWordBreakingData LocaleSearchData { get; set; }
public SearchIndexProcessObjectParameters(long localeId, long objectID, AyaType objectType, string name)
public SearchIndexProcessObjectParameters(long localeId, long objectID, AyaType objectType, string name, LocaleWordBreakingData localeSearchData = null)
{
Words = new List<string>();
@@ -391,6 +392,7 @@ namespace AyaNova.Biz
ObjectId = objectID;
ObjectType = objectType;
Name = name;
LocaleSearchData = localeSearchData;
}
@@ -678,7 +680,8 @@ RETURNING id, xmin;
}
//Get the current stopwords for the user's locale
private static LocaleWordBreakingData GetLocaleSearchData(long localeId, AyContext ct = null)
//called in here in this class and also by any bulk ops like seeding etc
internal static LocaleWordBreakingData GetLocaleSearchData(long localeId, AyContext ct = null)
{
LocaleWordBreakingData LSD = new LocaleWordBreakingData();
if (ct == null)
@@ -722,35 +725,26 @@ RETURNING id, xmin;
/// Use Locale setting CJKIndex=true to handle Chinese, Japanese, Korean etc
/// (languages with no easily identifiable word boundaries as in english)
/// </summary>
///
/// <param name="localeId"></param>
/// <param name="textStrings">A stringlist of 0 to * strings of text</param>
/// <returns>List of strings</returns>
internal static List<string> Break(long localeId, List<string> textStrings)// params string[] text)
internal static List<string> Break(long localeId, List<string> textStrings, LocaleWordBreakingData LocaleSearchData = null)
{
return BreakCore(localeId, false, textStrings);
return BreakCore(localeId, false, textStrings, LocaleSearchData);
}
/// <summary>
///
/// </summary>
/// <param name="localeId"></param>
/// <param name="textString"></param>
/// <returns></returns>
internal static List<string> Break(long localeId, string textString)// params string[] text)
internal static List<string> Break(long localeId, string textString, LocaleWordBreakingData LocaleSearchData = null)// params string[] text)
{
List<string> textStrings = new List<string>(1);
textStrings.Add(textString);
return BreakCore(localeId, false, textStrings);
return BreakCore(localeId, false, textStrings, LocaleSearchData);
}
/// <summary>
/// Used to Process users search phrase and preserve wild
/// cards entered
/// </summary>
/// <param name="localeId"></param>
/// <param name="searchPhrase"></param>
/// <returns></returns>
internal static List<string> BreakSearchPhrase(long localeId, string searchPhrase)
{
List<string> textStrings = new List<string>();
@@ -764,10 +758,12 @@ RETURNING id, xmin;
/// </summary>
// public static System.Collections.Generic.List<string> StopList = null;
internal static List<string> BreakCore(long localeId, bool KeepWildCards, List<string> textStrings)
internal static List<string> BreakCore(long localeId, bool KeepWildCards, List<string> textStrings, LocaleWordBreakingData LocaleSearchData = null)
{
//Get stopwords and CJKIndex flag value
LocaleWordBreakingData LocaleSearchData = GetLocaleSearchData(localeId);
//For stopwords and CJKIndex flag value
//if not provided (will be provided by seeder for performance but normally never) then fetch
if (LocaleSearchData == null)
LocaleSearchData = GetLocaleSearchData(localeId);
int MAXWORDLENGTH = 255;
int MINWORDLENGTH = 2;//A word isn't a word unless it's got at least two characters in it
StringBuilder sbResults = new StringBuilder();

11
server/AyaNova/biz/WidgetBiz.cs
View File

@@ -65,9 +65,8 @@ namespace AyaNova.Biz
////////////////////////////////////////////////////////////////////////////////////////////////
//CREATE
//route linked version for external api access
internal Widget Create(Widget inObj)
//Called from route and also seeder
internal Widget Create(Widget inObj, Search.LocaleWordBreakingData LocaleSearchData = null)
{
Validate(inObj, null);
if (HasErrors)
@@ -94,7 +93,7 @@ namespace AyaNova.Biz
//This takes 16 seconds out of 22 when seeding 500 widgets
SearchIndex(outObj, true);
SearchIndex(outObj, true, LocaleSearchData);
//This takes 2 seconds out of 22 when seeding 500 widgets
@@ -228,10 +227,10 @@ namespace AyaNova.Biz
}
private void SearchIndex(Widget obj, bool isNew)
private void SearchIndex(Widget obj, bool isNew, Search.LocaleWordBreakingData LocaleSearchData=null)
{
//SEARCH INDEXING
var SearchParams = new Search.SearchIndexProcessObjectParameters(UserLocaleId, obj.Id, BizType, obj.Name);
var SearchParams = new Search.SearchIndexProcessObjectParameters(UserLocaleId, obj.Id, BizType, obj.Name, LocaleSearchData);
SearchParams.AddText(obj.Notes).AddText(obj.Name).AddText(obj.Serial).AddText(obj.Tags).AddCustomFields(obj.CustomFields);

10
server/AyaNova/util/Seeder.cs
View File

@@ -17,6 +17,8 @@ namespace AyaNova.Util
public enum SeedLevel { SmallOneManShopTrialDataSet, MediumLocalServiceCompanyTrialDataSet, LargeCorporateMultiRegionalTrialDataSet, HugeForLoadTest };
public static int SeededUserCount = 0;
//cache the locale word breaking data
private static Search.LocaleWordBreakingData LocaleSearchData = null;
//////////////////////////////////////////////////////
@@ -73,6 +75,12 @@ namespace AyaNova.Util
using (var cct = ServiceProviderProvider.DBContext)
{
//Get a cached LocaleWordBreakingData to speed up generation
LocaleSearchData = Search.GetLocaleSearchData(ServerBootConfig.AYANOVA_DEFAULT_LANGUAGE_ID, cct);
}
//Set the time zone of the manager account
using (var cct = ServiceProviderProvider.DBContext)
{
@@ -622,7 +630,7 @@ namespace AyaNova.Util
//var NewObject = Cached_WidgetBiz.CreateAsync(o).Result;
//test without cached widgetbiz
WidgetBiz biz = WidgetBiz.GetBizInternal(ServiceProviderProvider.DBContext);
var NewObject = biz.Create(o);
var NewObject = biz.Create(o, LocaleSearchData);
if (NewObject == null)
{
log.LogError($"Seeder::GenSeedWidget error creating widget {o.Name}\r\n" + biz.GetErrorsAsString());