admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

after perf improvements and experimentation

Browse Source
This commit is contained in:
John Cardinal
2020-01-24 01:05:16 +00:00
parent 8edeae4c04
commit cf79c5e90c
4 changed files with 124 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/AyaNova/Startup.cs
View File

@@ -449,7 +449,7 @@ namespace AyaNova
{
AyaNova.Core.License.Fetch(apiServerState, dbContext, _newLog);
//NOTE: For unit testing make sure the time zone in util is set to the same figure as here to ensure list filter by date tests will work because server is on same page as user in terms of time
Util.Seeder.SeedDatabase(Util.Seeder.SeedLevel.MediumLocalServiceCompanyTrialDataSet, -7).Wait();//#############################################################################################
Util.Seeder.SeedDatabase(Util.Seeder.SeedLevel.MediumLocalServiceCompanyTrialDataSet, -7);//.Wait();//#############################################################################################
}
//TESTING
#endif

20
server/AyaNova/biz/WidgetBiz.cs
View File

@@ -87,13 +87,27 @@ namespace AyaNova.Biz
var TheContext = ct;
if (TheContext == null)
TheContext = ServiceProviderProvider.DBContext;
await TheContext.Widget.AddAsync(outObj);
await TheContext.SaveChangesAsync();
//TEST SPEED IMPROVEMENTS
//ORIGINALLY
// await TheContext.Widget.AddAsync(outObj);
// await TheContext.SaveChangesAsync();
//add syncronously and don't save but let the log save
//THIS SAVED 1 SECOND OUT OF 22 for seeding 500 widgets
TheContext.Widget.Add(outObj);
//Handle child and associated items:
EventLogProcessor.LogEventToDatabaseAndSaveEntireContext(new Event(UserId, outObj.Id, BizType, AyaEvent.Created), TheContext);
//This takes 16 seconds out of 22 when seeding 500 widgets
SearchIndex(outObj, true);
TagUtil.ProcessUpdateTagsInRepository(TheContext, outObj.Tags, null);
//This takes 2 seconds out of 22 when seeding 500 widgets
TagUtil.ProcessUpdateTagsInRepository(TheContext, outObj.Tags, null);
return outObj;
}
}

179
server/AyaNova/util/Seeder.cs
View File

@@ -7,7 +7,6 @@ using Microsoft.EntityFrameworkCore;
using Bogus;
using AyaNova.Api.ControllerHelpers;
using System.Diagnostics;
using System.Threading.Tasks;
namespace AyaNova.Util
@@ -18,48 +17,20 @@ namespace AyaNova.Util
public enum SeedLevel { SmallOneManShopTrialDataSet, MediumLocalServiceCompanyTrialDataSet, LargeCorporateMultiRegionalTrialDataSet, HugeForLoadTest };
public static int SeededUserCount = 0;
#region CACHED OBJECTS
//DB CONTEXT
private static AyContext _context = null;
internal static AyContext Cached_Context
{
get
{
if (_context == null)
{
_context = ServiceProviderProvider.DBContext;
}
return _context;
}
}
//WIDGETBIZ
private static WidgetBiz _widgetBiz = null;
internal static WidgetBiz Cached_WidgetBiz
{
get
{
if (_widgetBiz == null)
{
_widgetBiz = WidgetBiz.GetBizInternal(Cached_Context);
}
return _widgetBiz;
}
}
#endregion cached objects
//////////////////////////////////////////////////////
//Seed database for trial and testing purposes
//
public async static Task SeedDatabase(SeedLevel slevel, Decimal timeZoneOffset)
public static void SeedDatabase(SeedLevel slevel, Decimal timeZoneOffset)
{
await SeedDatabase(slevel, Guid.Empty, timeZoneOffset);
SeedDatabase(slevel, Guid.Empty, timeZoneOffset);
}
public async static Task SeedDatabase(SeedLevel slevel, Guid JobId, Decimal timeZoneOffset)
//public async static Task SeedDatabase(SeedLevel slevel, Guid JobId, Decimal timeZoneOffset)
public static void SeedDatabase(SeedLevel slevel, Guid JobId, Decimal timeZoneOffset)
{
bool LogJob = JobId != Guid.Empty;
SeededUserCount = 0;
@@ -103,10 +74,11 @@ namespace AyaNova.Util
//Set the time zone of the manager account
using (var cct = ServiceProviderProvider.DBContext)
{
var mgr = Cached_Context.UserOptions.FirstAsync(m => m.Id == 1).Result;
var mgr = cct.UserOptions.FirstAsync(m => m.Id == 1).Result;
mgr.TimeZoneOffset = timeZoneOffset;
Cached_Context.SaveChanges();
cct.SaveChanges();
}
@@ -160,7 +132,10 @@ namespace AyaNova.Util
};
//Create and save to db
var res = FormCustomBiz.GetBizInternal(Cached_Context).CreateAsync(fc).Result;
using (var cct = ServiceProviderProvider.DBContext)
{
FormCustomBiz.GetBizInternal(cct).CreateAsync(fc).Wait();
}
}
//Seed special test data for integration testing
@@ -191,7 +166,8 @@ namespace AyaNova.Util
//100 widgets
watch = new Stopwatch();
watch.Start();
await GenSeedWidgetAsync(log, 100);
//await GenSeedWidgetAsync(log, 100);
GenSeedWidget(log, 100);
//PERF
watch.Stop();
LogStatus(JobId, LogJob, log, $"100 Widgets seeded in {watch.ElapsedMilliseconds} ms");
@@ -252,7 +228,8 @@ namespace AyaNova.Util
watch = new Stopwatch();
watch.Start();
await GenSeedWidgetAsync(log, 500);
//await GenSeedWidgetAsync(log, 500);
GenSeedWidget(log, 500);
//PERF
watch.Stop();
LogStatus(JobId, LogJob, log, $"500 Widgets seeded in {watch.ElapsedMilliseconds} ms");
@@ -327,7 +304,8 @@ namespace AyaNova.Util
LogStatus(JobId, LogJob, log, $"Seeding 5,000 Widgets....");
watch = new Stopwatch();
watch.Start();
await GenSeedWidgetAsync(log, 5000);
//await GenSeedWidgetAsync(log, 5000);
GenSeedWidget(log, 5000);
//PERF
watch.Stop();
@@ -403,7 +381,8 @@ namespace AyaNova.Util
LogStatus(JobId, LogJob, log, $"Seeding 20,000 Widgets....");
watch = new Stopwatch();
watch.Start();
await GenSeedWidgetAsync(log, 20000);
//await GenSeedWidgetAsync(log, 20000);
GenSeedWidget(log, 20000);
watch.Stop();
LogStatus(JobId, LogJob, log, $"20k Widgets seeded in {watch.ElapsedMilliseconds} ms");
#endregion genhuge
@@ -424,8 +403,6 @@ namespace AyaNova.Util
{
log.LogInformation($"Seeder: setting server state back to {wasServerState.ToString()}");
apiServerState.SetState(wasServerState, wasReason);
_context = null;
_widgetBiz = null;
}
}
@@ -534,50 +511,52 @@ namespace AyaNova.Util
public static void GenSeedUser(ILogger log, int count, AuthorizationRoles roles, UserType userType, decimal timeZoneOffset, bool active = true, string login = null, string password = null, long localeId = 0)
{
UserBiz Biz = UserBiz.GetBizInternal(Cached_Context);
//allow creation of not entirely ready users (missing client id or subcontractor vendor id etc)
Biz.SeedOrImportRelaxedRulesMode = true;
Faker Fake = new Faker();
for (int x = 0; x < count; x++)
using (var cct = ServiceProviderProvider.DBContext)
{
User u = new User();
u.Active = active;
UserBiz Biz = UserBiz.GetBizInternal(cct);
var p = new Bogus.Person();
u.Name = Uniquify(p.FullName);
if (login != null)
//allow creation of not entirely ready users (missing client id or subcontractor vendor id etc)
Biz.SeedOrImportRelaxedRulesMode = true;
Faker Fake = new Faker();
for (int x = 0; x < count; x++)
{
u.Login = login;
u.Name += " - " + login;
}
else
u.Login = p.FirstName;
User u = new User();
u.Active = active;
if (password != null)
u.Password = password;
else
u.Password = u.Login;
u.Roles = roles;
u.LocaleId = localeId == 0 ? ServerBootConfig.AYANOVA_DEFAULT_LANGUAGE_ID : localeId;
u.UserType = userType;
u.EmployeeNumber = "A-" + (454 + SeededUserCount).ToString() + "-Y";
u.Notes = Fake.Lorem.Paragraph(2);
//TODO: After have USER and HEADOFFICE and VENDOR, if usertype is subcontractor or client or headoffice it needs to set a corresponding user's parent org record id to go with it
u.Tags = RandomTags(Fake);
//Children and relations
u.UserOptions = new UserOptions();
u.UserOptions.EmailAddress = p.Email.Replace("gmail.com", "helloayanova.com").Replace("hotmail.com", "helloayanova.com").Replace("yahoo.com", "helloayanova.com");
u.UserOptions.TimeZoneOffset = timeZoneOffset;
var p = new Bogus.Person();
u.Name = Uniquify(p.FullName);
if (login != null)
{
u.Login = login;
u.Name += " - " + login;
}
else
u.Login = p.FirstName;
var NewObject = Biz.Create(Cached_Context, u);
if (NewObject == null)
{
log.LogError($"Seeder::GenSeedUser error creating user {u.Name}\r\n" + Biz.GetErrorsAsString());
throw new System.Exception("Seeder::GenSeedUser error creating user\r\n" + Biz.GetErrorsAsString());
if (password != null)
u.Password = password;
else
u.Password = u.Login;
u.Roles = roles;
u.LocaleId = localeId == 0 ? ServerBootConfig.AYANOVA_DEFAULT_LANGUAGE_ID : localeId;
u.UserType = userType;
u.EmployeeNumber = "A-" + (454 + SeededUserCount).ToString() + "-Y";
u.Notes = Fake.Lorem.Sentence();//Fake.Lorem.Paragraph(2);
//TODO: After have USER and HEADOFFICE and VENDOR, if usertype is subcontractor or client or headoffice it needs to set a corresponding user's parent org record id to go with it
u.Tags = RandomTags(Fake);
//Children and relations
u.UserOptions = new UserOptions();
u.UserOptions.EmailAddress = p.Email.Replace("gmail.com", "helloayanova.com").Replace("hotmail.com", "helloayanova.com").Replace("yahoo.com", "helloayanova.com");
u.UserOptions.TimeZoneOffset = timeZoneOffset;
var NewObject = Biz.Create(cct, u);
if (NewObject == null)
{
log.LogError($"Seeder::GenSeedUser error creating user {u.Name}\r\n" + Biz.GetErrorsAsString());
throw new System.Exception("Seeder::GenSeedUser error creating user\r\n" + Biz.GetErrorsAsString());
}
}
}
@@ -587,8 +566,24 @@ namespace AyaNova.Util
//////////////////////////////////////////////////////
//Seed widget for testing
//
public static async Task GenSeedWidgetAsync(ILogger log, int count)
public static void GenSeedWidget(ILogger log, int count)
{
//COMMIT BEFORE GOING NON ASYNC AGAIN TO SEE IF MEDIUM 500 WIDGETS STILL TAKES SO LONG:
//ASYNC CACHED STUFF 2020-01-23 15:58:31.1619|INFO|Seeder|500 Widgets seeded in 54890 ms
//NON ASYNC ORIGINAL METHOD 2020-01-23 16:19:59.9131|INFO|Seeder|500 Widgets seeded in 51442 ms
//NON ASYNC HERE BUT CALLING ASYNC CREATE IN WIDGET: 2020-01-23 16:23:58.9478|INFO|Seeder|500 Widgets seeded in 53362 ms
//NOT CALLING SAVE ON WIDGET BUT DOING ALL OTHER CODE: 2020-01-23 16:25:45.0293|INFO|Seeder|500 Widgets seeded in 30 ms
//So slowness is all in the save code, let's see whats what there...
// first tried not caching teh widgetbiz object: 2020-01-23 16:28:58.0876|INFO|Seeder|500 Widgets seeded in 53577 ms
//so that's pointless to cache it
//ONE LAST THING IN HERE, trying *NOT* caching the db context and create each time: 2020-01-23 16:32:13.8966|INFO|Seeder|500 Widgets seeded in 22679 ms
// - Holy shit, that's back to where it was a few days ago, so for some reason using the same context over and over is slowing shit down drastically
// ripping that out now.
//OK, making changes in CREATE code in widgetbiz
//First removed extra save call and async db access shaved off a second
//the commented out the Search indexer and it dropped from 22 seconds to 4.5!!!!
//So the search indexing code is super fucking slow 2020-01-23 16:45:20.9518|INFO|Seeder|500 Widgets seeded in 4572 ms
var f = new Bogus.Faker();
@@ -608,14 +603,15 @@ namespace AyaNova.Util
//Random but valid enum
AuthorizationRoles randomRole = (AuthorizationRoles)values.GetValue(random.Next(values.Length));
o.Roles = randomRole;
o.Notes = f.Lorem.Paragraphs();
o.Notes = f.Lorem.Sentence();
o.Tags = RandomTags(f);
o.UserId = f.Random.Int(1, SeededUserCount);
//RANDOM CUSTOM FIELD DATA
var c1 = DateUtil.UniversalISO8661Format(f.Date.Between(DateTime.Now.AddYears(-1), DateTime.Now.AddYears(1)));
var c2 = f.Lorem.Paragraph();
var c2 = f.Lorem.Sentence();
var c3 = f.Random.Int(1, 99999999);
var c4 = f.Random.Bool().ToString().ToLowerInvariant();
var c5 = f.Random.Decimal();
@@ -623,13 +619,16 @@ namespace AyaNova.Util
o.CustomFields = $@"{{c1:""{c1}"",c2:""{c2}"",c3:{c3},c4:{c4},c5:{c5}}}";
var NewObject = await Cached_WidgetBiz.CreateAsync(o);
//var NewObject = Cached_WidgetBiz.CreateAsync(o).Result;
//test without cached widgetbiz
WidgetBiz biz = WidgetBiz.GetBizInternal(ServiceProviderProvider.DBContext);
var NewObject = biz.CreateAsync(o).Result;
if (NewObject == null)
{
log.LogError($"Seeder::GenSeedWidget error creating widget {o.Name}\r\n" + Cached_WidgetBiz.GetErrorsAsString());
throw new System.Exception("Seeder::GenSeedWidget error creating widget\r\n" + Cached_WidgetBiz.GetErrorsAsString());
{
log.LogError($"Seeder::GenSeedWidget error creating widget {o.Name}\r\n" + biz.GetErrorsAsString());
throw new System.Exception("Seeder::GenSeedWidget error creating widget\r\n" + biz.GetErrorsAsString());
}
}
}