admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Browse Source
This commit is contained in:
John Cardinal
2020-01-24 19:55:26 +00:00
parent 183e52202c
commit b241561fe6
6 changed files with 69 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
server/AyaNova/Startup.cs
View File

@@ -134,7 +134,7 @@ namespace AyaNova
bool LOG_SENSITIVE_DATA = false;
#if (DEBUG)
LOG_SENSITIVE_DATA = true;//############################################################################
LOG_SENSITIVE_DATA = false;//############################################################################
#endif
@@ -449,7 +449,7 @@ namespace AyaNova
{
AyaNova.Core.License.Fetch(apiServerState, dbContext, _newLog);
//NOTE: For unit testing make sure the time zone in util is set to the same figure as here to ensure list filter by date tests will work because server is on same page as user in terms of time
Util.Seeder.SeedDatabase(Util.Seeder.SeedLevel.SmallOneManShopTrialDataSet, -7);//#############################################################################################
Util.Seeder.SeedDatabase(Util.Seeder.SeedLevel.MediumLocalServiceCompanyTrialDataSet, -7);//#############################################################################################
}
//TESTING
#endif

46
server/AyaNova/biz/Search.cs
View File

@@ -380,10 +380,11 @@ namespace AyaNova.Biz
public AyaType ObjectType { get; set; }
public string Name { get; set; }
public List<string> Words { get; set; }
public LocaleWordBreakingData LocaleSearchData { get; set; }
public SearchIndexProcessObjectParameters(long localeId, long objectID, AyaType objectType, string name)
public SearchIndexProcessObjectParameters(long localeId, long objectID, AyaType objectType, string name, LocaleWordBreakingData localeSearchData = null)
{
Words = new List<string>();
@@ -391,6 +392,7 @@ namespace AyaNova.Biz
ObjectId = objectID;
ObjectType = objectType;
Name = name;
LocaleSearchData = localeSearchData;
}
@@ -433,7 +435,7 @@ namespace AyaNova.Biz
return this;
}
}
@@ -486,7 +488,7 @@ namespace AyaNova.Biz
//BREAK NAME STRING
List<string> NameKeyWordList = Break(p.LocaleId, p.Name);
//EARLY EXIT IF NO KEYWORDS OR NAME RECORD OR TAGS TO PROCESS
if (KeyWordList.Count == 0 && string.IsNullOrWhiteSpace(p.Name))
@@ -508,7 +510,7 @@ WHERE a.word IN ('eos', 'quia', 'voluptate', 'delectus', 'sapiente', 'omnis', 's
*/
//Put the matching keyword ID's into the list
foreach (KeyValuePair<long, string> K in ExistingKeywordMatches)
{
@@ -525,7 +527,7 @@ WHERE a.word IN ('eos', 'quia', 'voluptate', 'delectus', 'sapiente', 'omnis', 's
var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###");
#endif
//TODO: this foreach block needs to add all the words at once in a single range query rather than one word at a time
//TODO: this foreach block needs to add all the words at once in a single range query rather than one word at a time
foreach (string KeyWord in KeyWordList)
{
if (!ExistingKeywordMatches.ContainsValue(KeyWord))
@@ -678,7 +680,8 @@ RETURNING id, xmin;
}
//Get the current stopwords for the user's locale
private static LocaleWordBreakingData GetLocaleSearchData(long localeId, AyContext ct = null)
//called in here in this class and also by any bulk ops like seeding etc
internal static LocaleWordBreakingData GetLocaleSearchData(long localeId, AyContext ct = null)
{
LocaleWordBreakingData LSD = new LocaleWordBreakingData();
if (ct == null)
@@ -721,36 +724,27 @@ RETURNING id, xmin;
///
/// Use Locale setting CJKIndex=true to handle Chinese, Japanese, Korean etc
/// (languages with no easily identifiable word boundaries as in english)
/// </summary>
///
/// <param name="localeId"></param>
/// <param name="textStrings">A stringlist of 0 to * strings of text</param>
/// </summary>
/// <returns>List of strings</returns>
internal static List<string> Break(long localeId, List<string> textStrings)// params string[] text)
internal static List<string> Break(long localeId, List<string> textStrings, LocaleWordBreakingData LocaleSearchData = null)
{
return BreakCore(localeId, false, textStrings);
return BreakCore(localeId, false, textStrings, LocaleSearchData);
}
/// <summary>
///
/// </summary>
/// <param name="localeId"></param>
/// <param name="textString"></param>
/// <returns></returns>
internal static List<string> Break(long localeId, string textString)// params string[] text)
/// </summary>
internal static List<string> Break(long localeId, string textString, LocaleWordBreakingData LocaleSearchData = null)// params string[] text)
{
List<string> textStrings = new List<string>(1);
textStrings.Add(textString);
return BreakCore(localeId, false, textStrings);
return BreakCore(localeId, false, textStrings, LocaleSearchData);
}
/// <summary>
/// Used to Process users search phrase and preserve wild
/// cards entered
/// </summary>
/// <param name="localeId"></param>
/// <param name="searchPhrase"></param>
/// <returns></returns>
/// </summary>
internal static List<string> BreakSearchPhrase(long localeId, string searchPhrase)
{
List<string> textStrings = new List<string>();
@@ -764,10 +758,12 @@ RETURNING id, xmin;
/// </summary>
// public static System.Collections.Generic.List<string> StopList = null;
internal static List<string> BreakCore(long localeId, bool KeepWildCards, List<string> textStrings)
internal static List<string> BreakCore(long localeId, bool KeepWildCards, List<string> textStrings, LocaleWordBreakingData LocaleSearchData = null)
{
//Get stopwords and CJKIndex flag value
LocaleWordBreakingData LocaleSearchData = GetLocaleSearchData(localeId);
//For stopwords and CJKIndex flag value
//if not provided (will be provided by seeder for performance but normally never) then fetch
if (LocaleSearchData == null)
LocaleSearchData = GetLocaleSearchData(localeId);
int MAXWORDLENGTH = 255;
int MINWORDLENGTH = 2;//A word isn't a word unless it's got at least two characters in it
StringBuilder sbResults = new StringBuilder();

21
server/AyaNova/biz/WidgetBiz.cs
View File

@@ -65,9 +65,8 @@ namespace AyaNova.Biz
////////////////////////////////////////////////////////////////////////////////////////////////
//CREATE
//route linked version for external api access
internal Widget Create(Widget inObj)
//Called from route and also seeder
internal Widget Create(Widget inObj, Search.LocaleWordBreakingData LocaleSearchData = null)
{
Validate(inObj, null);
if (HasErrors)
@@ -84,7 +83,7 @@ namespace AyaNova.Biz
outObj.CustomFields = JsonUtil.CompactJson(outObj.CustomFields);
//Save to db
//add syncronously and don't save but let the log save
//THIS SAVED 1 SECOND OUT OF 22 for seeding 500 widgets
ct.Widget.Add(outObj);
@@ -94,11 +93,11 @@ namespace AyaNova.Biz
//This takes 16 seconds out of 22 when seeding 500 widgets
SearchIndex(outObj, true);
//This takes 2 seconds out of 22 when seeding 500 widgets
TagUtil.ProcessUpdateTagsInRepository(ct, outObj.Tags, null);
SearchIndex(outObj, true, LocaleSearchData);
//This takes 2 seconds out of 22 when seeding 500 widgets
TagUtil.ProcessUpdateTagsInRepository(ct, outObj.Tags, null);
return outObj;
}
@@ -228,10 +227,10 @@ namespace AyaNova.Biz
}
private void SearchIndex(Widget obj, bool isNew)
private void SearchIndex(Widget obj, bool isNew, Search.LocaleWordBreakingData LocaleSearchData=null)
{
//SEARCH INDEXING
var SearchParams = new Search.SearchIndexProcessObjectParameters(UserLocaleId, obj.Id, BizType, obj.Name);
var SearchParams = new Search.SearchIndexProcessObjectParameters(UserLocaleId, obj.Id, BizType, obj.Name, LocaleSearchData);
SearchParams.AddText(obj.Notes).AddText(obj.Name).AddText(obj.Serial).AddText(obj.Tags).AddCustomFields(obj.CustomFields);

14
server/AyaNova/util/Seeder.cs
View File

@@ -17,6 +17,8 @@ namespace AyaNova.Util
public enum SeedLevel { SmallOneManShopTrialDataSet, MediumLocalServiceCompanyTrialDataSet, LargeCorporateMultiRegionalTrialDataSet, HugeForLoadTest };
public static int SeededUserCount = 0;
//cache the locale word breaking data
private static Search.LocaleWordBreakingData LocaleSearchData = null;
//////////////////////////////////////////////////////
@@ -73,6 +75,12 @@ namespace AyaNova.Util
using (var cct = ServiceProviderProvider.DBContext)
{
//Get a cached LocaleWordBreakingData to speed up generation
LocaleSearchData = Search.GetLocaleSearchData(ServerBootConfig.AYANOVA_DEFAULT_LANGUAGE_ID, cct);
}
//Set the time zone of the manager account
using (var cct = ServiceProviderProvider.DBContext)
{
@@ -603,7 +611,7 @@ namespace AyaNova.Util
//Random but valid enum
AuthorizationRoles randomRole = (AuthorizationRoles)values.GetValue(random.Next(values.Length));
o.Roles = randomRole;
o.Notes = f.Lorem.Sentence();
o.Tags = RandomTags(f);
@@ -615,14 +623,14 @@ namespace AyaNova.Util
var c3 = f.Random.Int(1, 99999999);
var c4 = f.Random.Bool().ToString().ToLowerInvariant();
var c5 = f.Random.Decimal();
o.CustomFields = $@"{{c1:""{c1}"",c2:""{c2}"",c3:{c3},c4:{c4},c5:{c5}}}";
//var NewObject = Cached_WidgetBiz.CreateAsync(o).Result;
//test without cached widgetbiz
WidgetBiz biz = WidgetBiz.GetBizInternal(ServiceProviderProvider.DBContext);
var NewObject = biz.Create(o);
var NewObject = biz.Create(o, LocaleSearchData);
if (NewObject == null)
{
log.LogError($"Seeder::GenSeedWidget error creating widget {o.Name}\r\n" + biz.GetErrorsAsString());