From 41fcc81bed9a8fd27eb4b08614667de35020f967 Mon Sep 17 00:00:00 2001 From: John Cardinal Date: Tue, 2 Jun 2020 21:47:00 +0000 Subject: [PATCH] --- .vscode/launch.json | 4 +- .../Controllers/ServerMetricsController.cs | 2 +- server/AyaNova/util/AySchema.cs | 11 +- server/AyaNova/util/Seeder.cs | 116 +++++++++--------- 4 files changed, 68 insertions(+), 65 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 379d2046..ee6f9a8a 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -49,8 +49,8 @@ "AYANOVA_USE_URLS": "http://*:7575;", "AYANOVA_FOLDER_USER_FILES": "c:\\temp\\RavenTestData\\userfiles", "AYANOVA_FOLDER_BACKUP_FILES": "c:\\temp\\RavenTestData\\backupfiles", - "AYANOVA_SERVER_TEST_MODE":"false", - "AYANOVA_SERVER_TEST_MODE_SEEDLEVEL":"small", + "AYANOVA_SERVER_TEST_MODE":"true", + "AYANOVA_SERVER_TEST_MODE_SEEDLEVEL":"huge", "AYANOVA_SERVER_TEST_MODE_TZ_OFFSET":"-7", "AYANOVA_BACKUP_PG_DUMP_PATH":"C:\\data\\code\\PostgreSQLPortable_12.0\\App\\PgSQL\\bin\\" diff --git a/server/AyaNova/Controllers/ServerMetricsController.cs b/server/AyaNova/Controllers/ServerMetricsController.cs index 74ddd9e1..faec555c 100644 --- a/server/AyaNova/Controllers/ServerMetricsController.cs +++ b/server/AyaNova/Controllers/ServerMetricsController.cs @@ -247,7 +247,7 @@ namespace AyaNova.Api.Controllers } } - //trim out tables we don't want here + //trim out tables less than 1mb (the math above sets anything less than 1mb to zero) TopTables = TopTables.Where(z => z.value > 0).OrderByDescending(z => z.value).ToList(); long DBTotalSize = 0; diff --git a/server/AyaNova/util/AySchema.cs b/server/AyaNova/util/AySchema.cs index 9cd13c62..c7d9e94e 100644 --- a/server/AyaNova/util/AySchema.cs +++ b/server/AyaNova/util/AySchema.cs @@ -256,9 +256,10 @@ namespace AyaNova.Util //SEARCH TABLES - await ExecQueryAsync("CREATE TABLE asearchdictionary (id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, word varchar(255) not null)"); + await ExecQueryAsync("CREATE TABLE asearchdictionary (id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, word varchar(255) not null unique)"); //Must be unique and also this is hit a *lot* during searches and also indexing - await ExecQueryAsync("CREATE UNIQUE INDEX asearchdictionary_word_idx ON asearchdictionary (word);"); + //On actual testing this index is never used so for now removing it, perhaps it is a case of bad data but I tested with Huge dataset + //await ExecQueryAsync("CREATE UNIQUE INDEX asearchdictionary_word_idx ON asearchdictionary (word);"); //search key await ExecQueryAsync("CREATE TABLE asearchkey (id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, wordid bigint not null REFERENCES asearchdictionary (id), objectid bigint not null, objecttype integer not null)"); @@ -269,8 +270,10 @@ namespace AyaNova.Util //await ExecQueryAsync("CREATE INDEX asearchkey_typeid_idx ON asearchkey (objectid, objecttype );"); //This is what is needed during Searching - //search does a lot of hits on searchkey looking for the wordid and optionally objecttype - await ExecQueryAsync("CREATE INDEX asearchkey_wordid_otype_idx ON asearchkey (wordid, objecttype);"); + //search does a lot of hits on searchkey looking for the wordid and optionally objecttype + //In testing this did not pan out, in fact it was much faster to search both with and without a objecttype specified to simply have an index on wordid + // await ExecQueryAsync("CREATE INDEX asearchkey_wordid_otype_idx ON asearchkey (wordid, objecttype);"); + await ExecQueryAsync("CREATE INDEX asearchkey_wordid_idx ON asearchkey (wordid);"); //Search indexing stored procedure await ExecQueryAsync(@" diff --git a/server/AyaNova/util/Seeder.cs b/server/AyaNova/util/Seeder.cs index 202def7f..e09d034f 100644 --- a/server/AyaNova/util/Seeder.cs +++ b/server/AyaNova/util/Seeder.cs @@ -607,64 +607,7 @@ namespace AyaNova.Util u.UserOptions.CurrencyName = "USD"; u.UserOptions.UiColor = Fake.Internet.Color(); - - - //this seems wrong but is actually faster!? - UserBiz Biz = UserBiz.GetBiz(ServiceProviderProvider.DBContext); - //allow creation of not entirely ready users (missing client id or subcontractor vendor id etc) - Biz.SeedOrImportRelaxedRulesMode = true; - - - var NewObject = await Biz.CreateAsync(u); - if (NewObject == null) - { - log.LogError($"Seeder::GenSeedUser error creating user {u.Name}\r\n" + Biz.GetErrorsAsString()); - throw new System.Exception("Seeder::GenSeedUser error creating user\r\n" + Biz.GetErrorsAsString()); - } - //} - } - - SeededUserCount += count; - } - - ////////////////////////////////////////////////////// - //Seed widget for testing - // - public static async Task GenSeedWidgetAsync(ILogger log, int count) - { - //this is 4 times slower than doing it inside the loop below - //seems counterintuitive but maybe it's to do with the db context not being refreshed? - // WidgetBiz biz = WidgetBiz.GetBiz(ServiceProviderProvider.DBContext); - - var f = new Bogus.Faker(); - - - //RANDOM ROLES - Array values = Enum.GetValues(typeof(UserType)); - Random random = new Random(); - - DateTime seedStartWindow = DateTime.Now.AddYears(-1); - DateTime seedEndWindow = DateTime.Now.AddYears(1); - for (int x = 0; x < count; x++) - { - Widget o = new Widget(); - o.Name = Uniquify(f.Commerce.ProductName()); - o.Active = true; - // o.StartDate = f.Date.Between(DateTime.Now, DateTime.Now.AddMinutes(60)).ToUniversalTime(); - // o.EndDate = f.Date.Between(DateTime.Now.AddMinutes(90), DateTime.Now.AddHours(5)).ToUniversalTime(); - - // o.StartDate = DateTime.Now.ToUniversalTime(); - // o.EndDate = DateTime.Now.AddMinutes(60).ToUniversalTime(); - DateTime dtSeed = f.Date.Between(seedStartWindow, seedEndWindow).ToUniversalTime(); - o.StartDate = dtSeed; - o.EndDate = dtSeed.AddMinutes(60).ToUniversalTime(); - o.DollarAmount = Convert.ToDecimal(f.Commerce.Price()); - //Random but valid enum - UserType randomUserType = (UserType)values.GetValue(random.Next(values.Length)); - o.UserType = randomUserType; - - o.Notes = f.Lorem.Sentence(); - o.Wiki = @" + u.Wiki = @" # Markdown quick reference for Wiki pages *** @@ -787,6 +730,63 @@ A more detailed markdown guide is available here: *** "; + + //this seems wrong but is actually faster!? + UserBiz Biz = UserBiz.GetBiz(ServiceProviderProvider.DBContext); + //allow creation of not entirely ready users (missing client id or subcontractor vendor id etc) + Biz.SeedOrImportRelaxedRulesMode = true; + + + var NewObject = await Biz.CreateAsync(u); + if (NewObject == null) + { + log.LogError($"Seeder::GenSeedUser error creating user {u.Name}\r\n" + Biz.GetErrorsAsString()); + throw new System.Exception("Seeder::GenSeedUser error creating user\r\n" + Biz.GetErrorsAsString()); + } + //} + } + + SeededUserCount += count; + } + + ////////////////////////////////////////////////////// + //Seed widget for testing + // + public static async Task GenSeedWidgetAsync(ILogger log, int count) + { + //this is 4 times slower than doing it inside the loop below + //seems counterintuitive but maybe it's to do with the db context not being refreshed? + // WidgetBiz biz = WidgetBiz.GetBiz(ServiceProviderProvider.DBContext); + + var f = new Bogus.Faker(); + + + //RANDOM ROLES + Array values = Enum.GetValues(typeof(UserType)); + Random random = new Random(); + + DateTime seedStartWindow = DateTime.Now.AddYears(-1); + DateTime seedEndWindow = DateTime.Now.AddYears(1); + for (int x = 0; x < count; x++) + { + Widget o = new Widget(); + o.Name = Uniquify(f.Commerce.ProductName()); + o.Active = true; + // o.StartDate = f.Date.Between(DateTime.Now, DateTime.Now.AddMinutes(60)).ToUniversalTime(); + // o.EndDate = f.Date.Between(DateTime.Now.AddMinutes(90), DateTime.Now.AddHours(5)).ToUniversalTime(); + + // o.StartDate = DateTime.Now.ToUniversalTime(); + // o.EndDate = DateTime.Now.AddMinutes(60).ToUniversalTime(); + DateTime dtSeed = f.Date.Between(seedStartWindow, seedEndWindow).ToUniversalTime(); + o.StartDate = dtSeed; + o.EndDate = dtSeed.AddMinutes(60).ToUniversalTime(); + o.DollarAmount = Convert.ToDecimal(f.Commerce.Price()); + //Random but valid enum + UserType randomUserType = (UserType)values.GetValue(random.Next(values.Length)); + o.UserType = randomUserType; + + o.Notes = f.Lorem.Sentence(); + o.Tags = RandomTags(f); o.UserId = f.Random.Int(1, SeededUserCount);