admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Browse Source
This commit is contained in:
John Cardinal
2020-01-24 20:35:28 +00:00
parent b241561fe6
commit 008ea40649
5 changed files with 96 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
devdocs/todo.txt
View File

@@ -48,12 +48,18 @@ Async the keyword processing
Removing use of EF entirely in search indexing processing in favor of direct sql queries
cache or provide directly the locale to save time repeatedly fetching it when doing bulk ops
cache or provide directly the locale to save time repeatedly fetching it when doing bulk ops!!!
-After doing this 500=21 That's as fast as when I stripped out all the text, what a huge overhead saving right there!:
2020-01-24 12:00:41.2547|INFO|Seeder|Seeding 500 Widgets....
2020-01-24 12:00:51.9138|INFO|Seeder|500 Widgets seeded in 10649 ms
///////////////////////////////////////////////////////////////////////////////
TODO: Locale search data, there is overhead on every record save due to having to repeatedly fetch this
- While I added a way to seed it without fetching, there should actually just be a cached version for all record saving stored locally
- Fetches in the db are not necessary for this object more than once or on change of locale
- So probably Search code should cache it internally on first fetch
TODO: RUN data list query with debug info on and see if any EF stuff is creeping into it

32
server/AyaNova/biz/Search.cs
View File

@@ -19,7 +19,6 @@ namespace AyaNova.Biz
//This class handles word breaking, processing keywords and searching for results
public static class Search
{
//private static ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("Search");
#region Search and return results
@@ -380,11 +379,10 @@ namespace AyaNova.Biz
public AyaType ObjectType { get; set; }
public string Name { get; set; }
public List<string> Words { get; set; }
public LocaleWordBreakingData LocaleSearchData { get; set; }
public SearchIndexProcessObjectParameters(long localeId, long objectID, AyaType objectType, string name, LocaleWordBreakingData localeSearchData = null)
public SearchIndexProcessObjectParameters(long localeId, long objectID, AyaType objectType, string name)
{
Words = new List<string>();
@@ -392,7 +390,6 @@ namespace AyaNova.Biz
ObjectId = objectID;
ObjectType = objectType;
Name = name;
LocaleSearchData = localeSearchData;
}
@@ -726,19 +723,19 @@ RETURNING id, xmin;
/// (languages with no easily identifiable word boundaries as in english)
/// </summary>
/// <returns>List of strings</returns>
internal static List<string> Break(long localeId, List<string> textStrings, LocaleWordBreakingData LocaleSearchData = null)
internal static List<string> Break(long localeId, List<string> textStrings)
{
return BreakCore(localeId, false, textStrings, LocaleSearchData);
return BreakCore(localeId, false, textStrings);
}
/// <summary>
///
/// </summary>
internal static List<string> Break(long localeId, string textString, LocaleWordBreakingData LocaleSearchData = null)// params string[] text)
internal static List<string> Break(long localeId, string textString)
{
List<string> textStrings = new List<string>(1);
textStrings.Add(textString);
return BreakCore(localeId, false, textStrings, LocaleSearchData);
return BreakCore(localeId, false, textStrings);
}
/// <summary>
@@ -756,14 +753,21 @@ RETURNING id, xmin;
/// Stop words list reset upon login or editing of localized text
/// used for eliminating noise words from search dictionary
/// </summary>
// public static System.Collections.Generic.List<string> StopList = null;
internal static List<string> BreakCore(long localeId, bool KeepWildCards, List<string> textStrings, LocaleWordBreakingData LocaleSearchData = null)
public static Dictionary<long, LocaleWordBreakingData> localeWordBreakingDataCache = new Dictionary<long, LocaleWordBreakingData>();
internal static List<string> BreakCore(long localeId, bool KeepWildCards, List<string> textStrings)
{
//For stopwords and CJKIndex flag value
//if not provided (will be provided by seeder for performance but normally never) then fetch
if (LocaleSearchData == null)
LocaleSearchData = GetLocaleSearchData(localeId);
if (!localeWordBreakingDataCache.ContainsKey(localeId))
{
localeWordBreakingDataCache.Add(localeId, GetLocaleSearchData(localeId));
}
var localeWordBreakData = localeWordBreakingDataCache[localeId];
int MAXWORDLENGTH = 255;
int MINWORDLENGTH = 2;//A word isn't a word unless it's got at least two characters in it
StringBuilder sbResults = new StringBuilder();
@@ -797,7 +801,7 @@ RETURNING id, xmin;
//get it as a character
char c = t.GetTextElement()[0];
if (!LocaleSearchData.CJKIndex)
if (!localeWordBreakData.CJKIndex)
{
#region regular tokenizer
@@ -1027,7 +1031,7 @@ RETURNING id, xmin;
if (s.Length >= MINWORDLENGTH || (KeepWildCards && s.Contains('%')))
{
//Add only non stopwords
if (!LocaleSearchData.StopWords.Contains(s))
if (!localeWordBreakData.StopWords.Contains(s))
{
ReturnList.Add(s);
}

8
server/AyaNova/biz/WidgetBiz.cs
View File

@@ -66,7 +66,7 @@ namespace AyaNova.Biz
//CREATE
//Called from route and also seeder
internal Widget Create(Widget inObj, Search.LocaleWordBreakingData LocaleSearchData = null)
internal Widget Create(Widget inObj)
{
Validate(inObj, null);
if (HasErrors)
@@ -93,7 +93,7 @@ namespace AyaNova.Biz
//This takes 16 seconds out of 22 when seeding 500 widgets
SearchIndex(outObj, true, LocaleSearchData);
SearchIndex(outObj, true);
//This takes 2 seconds out of 22 when seeding 500 widgets
@@ -227,10 +227,10 @@ namespace AyaNova.Biz
}
private void SearchIndex(Widget obj, bool isNew, Search.LocaleWordBreakingData LocaleSearchData=null)
private void SearchIndex(Widget obj, bool isNew)
{
//SEARCH INDEXING
var SearchParams = new Search.SearchIndexProcessObjectParameters(UserLocaleId, obj.Id, BizType, obj.Name, LocaleSearchData);
var SearchParams = new Search.SearchIndexProcessObjectParameters(UserLocaleId, obj.Id, BizType, obj.Name);
SearchParams.AddText(obj.Notes).AddText(obj.Name).AddText(obj.Serial).AddText(obj.Tags).AddCustomFields(obj.CustomFields);

12
server/AyaNova/util/Seeder.cs
View File

@@ -17,15 +17,12 @@ namespace AyaNova.Util
public enum SeedLevel { SmallOneManShopTrialDataSet, MediumLocalServiceCompanyTrialDataSet, LargeCorporateMultiRegionalTrialDataSet, HugeForLoadTest };
public static int SeededUserCount = 0;
//cache the locale word breaking data
private static Search.LocaleWordBreakingData LocaleSearchData = null;
//////////////////////////////////////////////////////
//Seed database for trial and testing purposes
//
public static void SeedDatabase(SeedLevel slevel, Decimal timeZoneOffset)
{
SeedDatabase(slevel, Guid.Empty, timeZoneOffset);
@@ -74,13 +71,6 @@ namespace AyaNova.Util
DbUtil.EmptyBizDataFromDatabaseForSeedingOrImporting(log);
using (var cct = ServiceProviderProvider.DBContext)
{
//Get a cached LocaleWordBreakingData to speed up generation
LocaleSearchData = Search.GetLocaleSearchData(ServerBootConfig.AYANOVA_DEFAULT_LANGUAGE_ID, cct);
}
//Set the time zone of the manager account
using (var cct = ServiceProviderProvider.DBContext)
{
@@ -630,7 +620,7 @@ namespace AyaNova.Util
//var NewObject = Cached_WidgetBiz.CreateAsync(o).Result;
//test without cached widgetbiz
WidgetBiz biz = WidgetBiz.GetBizInternal(ServiceProviderProvider.DBContext);
var NewObject = biz.Create(o, LocaleSearchData);
var NewObject = biz.Create(o);
if (NewObject == null)
{
log.LogError($"Seeder::GenSeedWidget error creating widget {o.Name}\r\n" + biz.GetErrorsAsString());