admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

after perf improvements and experimentation

Browse Source
This commit is contained in:
John Cardinal
2020-01-24 01:05:16 +00:00
parent 8edeae4c04
commit cf79c5e90c
4 changed files with 124 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

179
server/AyaNova/util/Seeder.cs
View File

@@ -7,7 +7,6 @@ using Microsoft.EntityFrameworkCore;
using Bogus;
using AyaNova.Api.ControllerHelpers;
using System.Diagnostics;
using System.Threading.Tasks;
namespace AyaNova.Util
@@ -18,48 +17,20 @@ namespace AyaNova.Util
public enum SeedLevel { SmallOneManShopTrialDataSet, MediumLocalServiceCompanyTrialDataSet, LargeCorporateMultiRegionalTrialDataSet, HugeForLoadTest };
public static int SeededUserCount = 0;
#region CACHED OBJECTS
//DB CONTEXT
private static AyContext _context = null;
internal static AyContext Cached_Context
{
get
{
if (_context == null)
{
_context = ServiceProviderProvider.DBContext;
}
return _context;
}
}
//WIDGETBIZ
private static WidgetBiz _widgetBiz = null;
internal static WidgetBiz Cached_WidgetBiz
{
get
{
if (_widgetBiz == null)
{
_widgetBiz = WidgetBiz.GetBizInternal(Cached_Context);
}
return _widgetBiz;
}
}
#endregion cached objects
//////////////////////////////////////////////////////
//Seed database for trial and testing purposes
//
public async static Task SeedDatabase(SeedLevel slevel, Decimal timeZoneOffset)
public static void SeedDatabase(SeedLevel slevel, Decimal timeZoneOffset)
{
await SeedDatabase(slevel, Guid.Empty, timeZoneOffset);
SeedDatabase(slevel, Guid.Empty, timeZoneOffset);
}
public async static Task SeedDatabase(SeedLevel slevel, Guid JobId, Decimal timeZoneOffset)
//public async static Task SeedDatabase(SeedLevel slevel, Guid JobId, Decimal timeZoneOffset)
public static void SeedDatabase(SeedLevel slevel, Guid JobId, Decimal timeZoneOffset)
{
bool LogJob = JobId != Guid.Empty;
SeededUserCount = 0;
@@ -103,10 +74,11 @@ namespace AyaNova.Util
//Set the time zone of the manager account
using (var cct = ServiceProviderProvider.DBContext)
{
var mgr = Cached_Context.UserOptions.FirstAsync(m => m.Id == 1).Result;
var mgr = cct.UserOptions.FirstAsync(m => m.Id == 1).Result;
mgr.TimeZoneOffset = timeZoneOffset;
Cached_Context.SaveChanges();
cct.SaveChanges();
}
@@ -160,7 +132,10 @@ namespace AyaNova.Util
};
//Create and save to db
var res = FormCustomBiz.GetBizInternal(Cached_Context).CreateAsync(fc).Result;
using (var cct = ServiceProviderProvider.DBContext)
{
FormCustomBiz.GetBizInternal(cct).CreateAsync(fc).Wait();
}
}
//Seed special test data for integration testing
@@ -191,7 +166,8 @@ namespace AyaNova.Util
//100 widgets
watch = new Stopwatch();
watch.Start();
await GenSeedWidgetAsync(log, 100);
//await GenSeedWidgetAsync(log, 100);
GenSeedWidget(log, 100);
//PERF
watch.Stop();
LogStatus(JobId, LogJob, log, $"100 Widgets seeded in {watch.ElapsedMilliseconds} ms");
@@ -252,7 +228,8 @@ namespace AyaNova.Util
watch = new Stopwatch();
watch.Start();
await GenSeedWidgetAsync(log, 500);
//await GenSeedWidgetAsync(log, 500);
GenSeedWidget(log, 500);
//PERF
watch.Stop();
LogStatus(JobId, LogJob, log, $"500 Widgets seeded in {watch.ElapsedMilliseconds} ms");
@@ -327,7 +304,8 @@ namespace AyaNova.Util
LogStatus(JobId, LogJob, log, $"Seeding 5,000 Widgets....");
watch = new Stopwatch();
watch.Start();
await GenSeedWidgetAsync(log, 5000);
//await GenSeedWidgetAsync(log, 5000);
GenSeedWidget(log, 5000);
//PERF
watch.Stop();
@@ -403,7 +381,8 @@ namespace AyaNova.Util
LogStatus(JobId, LogJob, log, $"Seeding 20,000 Widgets....");
watch = new Stopwatch();
watch.Start();
await GenSeedWidgetAsync(log, 20000);
//await GenSeedWidgetAsync(log, 20000);
GenSeedWidget(log, 20000);
watch.Stop();
LogStatus(JobId, LogJob, log, $"20k Widgets seeded in {watch.ElapsedMilliseconds} ms");
#endregion genhuge
@@ -424,8 +403,6 @@ namespace AyaNova.Util
{
log.LogInformation($"Seeder: setting server state back to {wasServerState.ToString()}");
apiServerState.SetState(wasServerState, wasReason);
_context = null;
_widgetBiz = null;
}
}
@@ -534,50 +511,52 @@ namespace AyaNova.Util
public static void GenSeedUser(ILogger log, int count, AuthorizationRoles roles, UserType userType, decimal timeZoneOffset, bool active = true, string login = null, string password = null, long localeId = 0)
{
UserBiz Biz = UserBiz.GetBizInternal(Cached_Context);
//allow creation of not entirely ready users (missing client id or subcontractor vendor id etc)
Biz.SeedOrImportRelaxedRulesMode = true;
Faker Fake = new Faker();
for (int x = 0; x < count; x++)
using (var cct = ServiceProviderProvider.DBContext)
{
User u = new User();
u.Active = active;
UserBiz Biz = UserBiz.GetBizInternal(cct);
var p = new Bogus.Person();
u.Name = Uniquify(p.FullName);
if (login != null)
//allow creation of not entirely ready users (missing client id or subcontractor vendor id etc)
Biz.SeedOrImportRelaxedRulesMode = true;
Faker Fake = new Faker();
for (int x = 0; x < count; x++)
{
u.Login = login;
u.Name += " - " + login;
}
else
u.Login = p.FirstName;
User u = new User();
u.Active = active;
if (password != null)
u.Password = password;
else
u.Password = u.Login;
u.Roles = roles;
u.LocaleId = localeId == 0 ? ServerBootConfig.AYANOVA_DEFAULT_LANGUAGE_ID : localeId;
u.UserType = userType;
u.EmployeeNumber = "A-" + (454 + SeededUserCount).ToString() + "-Y";
u.Notes = Fake.Lorem.Paragraph(2);
//TODO: After have USER and HEADOFFICE and VENDOR, if usertype is subcontractor or client or headoffice it needs to set a corresponding user's parent org record id to go with it
u.Tags = RandomTags(Fake);
//Children and relations
u.UserOptions = new UserOptions();
u.UserOptions.EmailAddress = p.Email.Replace("gmail.com", "helloayanova.com").Replace("hotmail.com", "helloayanova.com").Replace("yahoo.com", "helloayanova.com");
u.UserOptions.TimeZoneOffset = timeZoneOffset;
var p = new Bogus.Person();
u.Name = Uniquify(p.FullName);
if (login != null)
{
u.Login = login;
u.Name += " - " + login;
}
else
u.Login = p.FirstName;
var NewObject = Biz.Create(Cached_Context, u);
if (NewObject == null)
{
log.LogError($"Seeder::GenSeedUser error creating user {u.Name}\r\n" + Biz.GetErrorsAsString());
throw new System.Exception("Seeder::GenSeedUser error creating user\r\n" + Biz.GetErrorsAsString());
if (password != null)
u.Password = password;
else
u.Password = u.Login;
u.Roles = roles;
u.LocaleId = localeId == 0 ? ServerBootConfig.AYANOVA_DEFAULT_LANGUAGE_ID : localeId;
u.UserType = userType;
u.EmployeeNumber = "A-" + (454 + SeededUserCount).ToString() + "-Y";
u.Notes = Fake.Lorem.Sentence();//Fake.Lorem.Paragraph(2);
//TODO: After have USER and HEADOFFICE and VENDOR, if usertype is subcontractor or client or headoffice it needs to set a corresponding user's parent org record id to go with it
u.Tags = RandomTags(Fake);
//Children and relations
u.UserOptions = new UserOptions();
u.UserOptions.EmailAddress = p.Email.Replace("gmail.com", "helloayanova.com").Replace("hotmail.com", "helloayanova.com").Replace("yahoo.com", "helloayanova.com");
u.UserOptions.TimeZoneOffset = timeZoneOffset;
var NewObject = Biz.Create(cct, u);
if (NewObject == null)
{
log.LogError($"Seeder::GenSeedUser error creating user {u.Name}\r\n" + Biz.GetErrorsAsString());
throw new System.Exception("Seeder::GenSeedUser error creating user\r\n" + Biz.GetErrorsAsString());
}
}
}
@@ -587,8 +566,24 @@ namespace AyaNova.Util
//////////////////////////////////////////////////////
//Seed widget for testing
//
public static async Task GenSeedWidgetAsync(ILogger log, int count)
public static void GenSeedWidget(ILogger log, int count)
{
//COMMIT BEFORE GOING NON ASYNC AGAIN TO SEE IF MEDIUM 500 WIDGETS STILL TAKES SO LONG:
//ASYNC CACHED STUFF 2020-01-23 15:58:31.1619|INFO|Seeder|500 Widgets seeded in 54890 ms
//NON ASYNC ORIGINAL METHOD 2020-01-23 16:19:59.9131|INFO|Seeder|500 Widgets seeded in 51442 ms
//NON ASYNC HERE BUT CALLING ASYNC CREATE IN WIDGET: 2020-01-23 16:23:58.9478|INFO|Seeder|500 Widgets seeded in 53362 ms
//NOT CALLING SAVE ON WIDGET BUT DOING ALL OTHER CODE: 2020-01-23 16:25:45.0293|INFO|Seeder|500 Widgets seeded in 30 ms
//So slowness is all in the save code, let's see whats what there...
// first tried not caching teh widgetbiz object: 2020-01-23 16:28:58.0876|INFO|Seeder|500 Widgets seeded in 53577 ms
//so that's pointless to cache it
//ONE LAST THING IN HERE, trying *NOT* caching the db context and create each time: 2020-01-23 16:32:13.8966|INFO|Seeder|500 Widgets seeded in 22679 ms
// - Holy shit, that's back to where it was a few days ago, so for some reason using the same context over and over is slowing shit down drastically
// ripping that out now.
//OK, making changes in CREATE code in widgetbiz
//First removed extra save call and async db access shaved off a second
//the commented out the Search indexer and it dropped from 22 seconds to 4.5!!!!
//So the search indexing code is super fucking slow 2020-01-23 16:45:20.9518|INFO|Seeder|500 Widgets seeded in 4572 ms
var f = new Bogus.Faker();
@@ -608,14 +603,15 @@ namespace AyaNova.Util
//Random but valid enum
AuthorizationRoles randomRole = (AuthorizationRoles)values.GetValue(random.Next(values.Length));
o.Roles = randomRole;
o.Notes = f.Lorem.Paragraphs();
o.Notes = f.Lorem.Sentence();
o.Tags = RandomTags(f);
o.UserId = f.Random.Int(1, SeededUserCount);
//RANDOM CUSTOM FIELD DATA
var c1 = DateUtil.UniversalISO8661Format(f.Date.Between(DateTime.Now.AddYears(-1), DateTime.Now.AddYears(1)));
var c2 = f.Lorem.Paragraph();
var c2 = f.Lorem.Sentence();
var c3 = f.Random.Int(1, 99999999);
var c4 = f.Random.Bool().ToString().ToLowerInvariant();
var c5 = f.Random.Decimal();
@@ -623,13 +619,16 @@ namespace AyaNova.Util
o.CustomFields = $@"{{c1:""{c1}"",c2:""{c2}"",c3:{c3},c4:{c4},c5:{c5}}}";
var NewObject = await Cached_WidgetBiz.CreateAsync(o);
//var NewObject = Cached_WidgetBiz.CreateAsync(o).Result;
//test without cached widgetbiz
WidgetBiz biz = WidgetBiz.GetBizInternal(ServiceProviderProvider.DBContext);
var NewObject = biz.CreateAsync(o).Result;
if (NewObject == null)
{
log.LogError($"Seeder::GenSeedWidget error creating widget {o.Name}\r\n" + Cached_WidgetBiz.GetErrorsAsString());
throw new System.Exception("Seeder::GenSeedWidget error creating widget\r\n" + Cached_WidgetBiz.GetErrorsAsString());
{
log.LogError($"Seeder::GenSeedWidget error creating widget {o.Name}\r\n" + biz.GetErrorsAsString());
throw new System.Exception("Seeder::GenSeedWidget error creating widget\r\n" + biz.GetErrorsAsString());
}
}
}