after perf improvements and experimentation

devdocs/todo.txt

@@ -5,6 +5,21 @@ eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOiIxNTcxODU5OTU0IiwiZXhwIjoiMTU3MjQ
 
 ## IMMEDIATE ITEMS
 
+PERFORMANCE CHANGE
+Baseline from before doing anything seeding a medium level with full text
+2020-01-21 16:49:17.4662|INFO|Seeder|75 Users seeded in 2279 ms
+2020-01-21 16:49:39.4481|INFO|Seeder|500 Widgets seeded in 21968 ms
+
+After round one of improvements (less text in seed data notes, not calling savechanges or add async)
+//about 2 seconds came from the async db stuff and the rest was from using less text so less indexing which isn't really a permanent solution just a minor one
+2020-01-23 16:57:57.0422|INFO|Seeder|75 Users seeded in 2398 ms
+2020-01-23 16:58:11.9983|INFO|Seeder|500 Widgets seeded in 14958 ms
+
+TODO: Search indexing is painfully slow, it accounts for 16 of 22 seconds when creating 500 widgets with full paragraphs of text 
+	- Try to see if it's just one part of the operation by timing it
+	- Re-code it not using EF but directly interacting with the DB
+	- Maybe it's a case for stored procedures or something?
+
 
 TODO: DataFilter how to distinguish between filtering on specific ID value or on value column
 	- Might need to add a filter on ID type of thing maybe?
@@ -18,6 +33,8 @@ UPDATE SPEC DOCS with new format and names etc or at least remove nonsense as ne
 
 UPDATE MANUAL API REGARDING LISTS
 
+
+
 TODO: Need route to gather all object role rights in a format useful to display in UI so that a biz manager can see at a glance the rights for different roles to objects	
 	- This way it's dynamic and picked up from the code itself which is always the source of truth so no need to put in the manual
 	- Would likely want to display multiple ways: for a specific object or role or selected user maybe too in the user info form ("effective roles")

server/AyaNova/Startup.cs

@@ -449,7 +449,7 @@ namespace AyaNova
             {
                 AyaNova.Core.License.Fetch(apiServerState, dbContext, _newLog);
                 //NOTE: For unit testing make sure the time zone in util is set to the same figure as here to ensure list filter by date tests will work because server is on same page as user in terms of time
-                Util.Seeder.SeedDatabase(Util.Seeder.SeedLevel.MediumLocalServiceCompanyTrialDataSet, -7).Wait();//#############################################################################################
+                Util.Seeder.SeedDatabase(Util.Seeder.SeedLevel.MediumLocalServiceCompanyTrialDataSet, -7);//.Wait();//#############################################################################################
             }
             //TESTING
 #endif

server/AyaNova/biz/WidgetBiz.cs

@@ -87,13 +87,27 @@ namespace AyaNova.Biz
                 var TheContext = ct;
                 if (TheContext == null)
                     TheContext = ServiceProviderProvider.DBContext;
-                await TheContext.Widget.AddAsync(outObj);
+                //TEST SPEED IMPROVEMENTS
-                await TheContext.SaveChangesAsync();
+
+                //ORIGINALLY
+                // await TheContext.Widget.AddAsync(outObj);
+                // await TheContext.SaveChangesAsync();
+
+                //add syncronously and don't save but let the log save
+                //THIS SAVED 1 SECOND OUT OF 22 for seeding 500 widgets
+                TheContext.Widget.Add(outObj);
 
                 //Handle child and associated items:                
                 EventLogProcessor.LogEventToDatabaseAndSaveEntireContext(new Event(UserId, outObj.Id, BizType, AyaEvent.Created), TheContext);
+
+
+                //This takes 16 seconds out of 22 when seeding 500 widgets
                 SearchIndex(outObj, true);
-                TagUtil.ProcessUpdateTagsInRepository(TheContext, outObj.Tags, null);
+              
+               
+               //This takes 2 seconds out of 22 when seeding 500 widgets
+               TagUtil.ProcessUpdateTagsInRepository(TheContext, outObj.Tags, null);
+
                 return outObj;
             }
         }

server/AyaNova/util/Seeder.cs

@@ -7,7 +7,6 @@ using Microsoft.EntityFrameworkCore;
 using Bogus;
 using AyaNova.Api.ControllerHelpers;
 using System.Diagnostics;
-using System.Threading.Tasks;
 
 
 namespace AyaNova.Util
@@ -18,48 +17,20 @@ namespace AyaNova.Util
         public enum SeedLevel { SmallOneManShopTrialDataSet, MediumLocalServiceCompanyTrialDataSet, LargeCorporateMultiRegionalTrialDataSet, HugeForLoadTest };
         public static int SeededUserCount = 0;
 
-        #region CACHED OBJECTS
 
-        //DB CONTEXT
-        private static AyContext _context = null;
-        internal static AyContext Cached_Context
-        {
-            get
-            {
-                if (_context == null)
-                {
-                    _context = ServiceProviderProvider.DBContext;
-                }
-                return _context;
-            }
-        }
-
-        //WIDGETBIZ
-        private static WidgetBiz _widgetBiz = null;
-        internal static WidgetBiz Cached_WidgetBiz
-        {
-            get
-            {
-                if (_widgetBiz == null)
-                {
-                    _widgetBiz = WidgetBiz.GetBizInternal(Cached_Context);
-                }
-                return _widgetBiz;
-            }
-        }
-        #endregion cached objects
 
         //////////////////////////////////////////////////////
         //Seed database for trial and testing purposes
         // 
 
-        public async static Task SeedDatabase(SeedLevel slevel, Decimal timeZoneOffset)
+
+        public static void SeedDatabase(SeedLevel slevel, Decimal timeZoneOffset)
         {
-            await SeedDatabase(slevel, Guid.Empty, timeZoneOffset);
+            SeedDatabase(slevel, Guid.Empty, timeZoneOffset);
         }
 
-
+        //public async static Task SeedDatabase(SeedLevel slevel, Guid JobId, Decimal timeZoneOffset)
-        public async static Task SeedDatabase(SeedLevel slevel, Guid JobId, Decimal timeZoneOffset)
+        public static void SeedDatabase(SeedLevel slevel, Guid JobId, Decimal timeZoneOffset)
         {
             bool LogJob = JobId != Guid.Empty;
             SeededUserCount = 0;
@@ -103,10 +74,11 @@ namespace AyaNova.Util
 
 
                 //Set the time zone of the manager account 
+                using (var cct = ServiceProviderProvider.DBContext)
                 {
-                    var mgr = Cached_Context.UserOptions.FirstAsync(m => m.Id == 1).Result;
+                    var mgr = cct.UserOptions.FirstAsync(m => m.Id == 1).Result;
                     mgr.TimeZoneOffset = timeZoneOffset;
-                    Cached_Context.SaveChanges();
+                    cct.SaveChanges();
                 }
 
 
@@ -160,7 +132,10 @@ namespace AyaNova.Util
                     };
 
                     //Create and save to db
-                    var res = FormCustomBiz.GetBizInternal(Cached_Context).CreateAsync(fc).Result;
+                    using (var cct = ServiceProviderProvider.DBContext)
+                    {
+                        FormCustomBiz.GetBizInternal(cct).CreateAsync(fc).Wait();
+                    }
                 }
 
                 //Seed special test data for integration testing
@@ -191,7 +166,8 @@ namespace AyaNova.Util
                             //100 widgets
                             watch = new Stopwatch();
                             watch.Start();
-                            await GenSeedWidgetAsync(log, 100);
+                            //await GenSeedWidgetAsync(log, 100);
+                            GenSeedWidget(log, 100);
                             //PERF
                             watch.Stop();
                             LogStatus(JobId, LogJob, log, $"100 Widgets seeded in {watch.ElapsedMilliseconds} ms");
@@ -252,7 +228,8 @@ namespace AyaNova.Util
                             watch = new Stopwatch();
                             watch.Start();
 
-                            await GenSeedWidgetAsync(log, 500);
+                            //await GenSeedWidgetAsync(log, 500);
+                            GenSeedWidget(log, 500);
                             //PERF
                             watch.Stop();
                             LogStatus(JobId, LogJob, log, $"500 Widgets seeded in {watch.ElapsedMilliseconds} ms");
@@ -327,7 +304,8 @@ namespace AyaNova.Util
                             LogStatus(JobId, LogJob, log, $"Seeding 5,000 Widgets....");
                             watch = new Stopwatch();
                             watch.Start();
-                            await GenSeedWidgetAsync(log, 5000);
+                            //await GenSeedWidgetAsync(log, 5000);
+                            GenSeedWidget(log, 5000);
 
                             //PERF
                             watch.Stop();
@@ -403,7 +381,8 @@ namespace AyaNova.Util
                             LogStatus(JobId, LogJob, log, $"Seeding 20,000 Widgets....");
                             watch = new Stopwatch();
                             watch.Start();
-                            await GenSeedWidgetAsync(log, 20000);
+                            //await GenSeedWidgetAsync(log, 20000);
+                            GenSeedWidget(log, 20000);
                             watch.Stop();
                             LogStatus(JobId, LogJob, log, $"20k Widgets seeded in {watch.ElapsedMilliseconds} ms");
                             #endregion genhuge
@@ -424,8 +403,6 @@ namespace AyaNova.Util
             {
                 log.LogInformation($"Seeder: setting server state back to {wasServerState.ToString()}");
                 apiServerState.SetState(wasServerState, wasReason);
-                _context = null;
-                _widgetBiz = null;
             }
         }
 
@@ -534,50 +511,52 @@ namespace AyaNova.Util
 
         public static void GenSeedUser(ILogger log, int count, AuthorizationRoles roles, UserType userType, decimal timeZoneOffset, bool active = true, string login = null, string password = null, long localeId = 0)
         {
-
+            using (var cct = ServiceProviderProvider.DBContext)
-            UserBiz Biz = UserBiz.GetBizInternal(Cached_Context);
-
-            //allow creation of not entirely ready users (missing client id or subcontractor vendor id etc)
-            Biz.SeedOrImportRelaxedRulesMode = true;
-
-            Faker Fake = new Faker();
-
-            for (int x = 0; x < count; x++)
             {
-                User u = new User();
+                UserBiz Biz = UserBiz.GetBizInternal(cct);
-                u.Active = active;
 
-                var p = new Bogus.Person();
+                //allow creation of not entirely ready users (missing client id or subcontractor vendor id etc)
-                u.Name = Uniquify(p.FullName);
+                Biz.SeedOrImportRelaxedRulesMode = true;
-                if (login != null)
+
+                Faker Fake = new Faker();
+
+                for (int x = 0; x < count; x++)
                 {
-                    u.Login = login;
+                    User u = new User();
-                    u.Name += " - " + login;
+                    u.Active = active;
-                }
-                else
-                    u.Login = p.FirstName;
 
-                if (password != null)
+                    var p = new Bogus.Person();
-                    u.Password = password;
+                    u.Name = Uniquify(p.FullName);
-                else
+                    if (login != null)
-                    u.Password = u.Login;
+                    {
-                u.Roles = roles;
+                        u.Login = login;
-                u.LocaleId = localeId == 0 ? ServerBootConfig.AYANOVA_DEFAULT_LANGUAGE_ID : localeId;
+                        u.Name += " - " + login;
-                u.UserType = userType;
+                    }
-                u.EmployeeNumber = "A-" + (454 + SeededUserCount).ToString() + "-Y";
+                    else
-                u.Notes = Fake.Lorem.Paragraph(2);
+                        u.Login = p.FirstName;
-                //TODO: After have USER and HEADOFFICE and VENDOR, if usertype is subcontractor or client or headoffice it needs to set a corresponding user's parent org record id to go with it
-                u.Tags = RandomTags(Fake);
-                //Children and relations
-                u.UserOptions = new UserOptions();
-                u.UserOptions.EmailAddress = p.Email.Replace("gmail.com", "helloayanova.com").Replace("hotmail.com", "helloayanova.com").Replace("yahoo.com", "helloayanova.com");
-                u.UserOptions.TimeZoneOffset = timeZoneOffset;
 
-                var NewObject = Biz.Create(Cached_Context, u);
+                    if (password != null)
-                if (NewObject == null)
+                        u.Password = password;
-                {
+                    else
-                    log.LogError($"Seeder::GenSeedUser error creating user {u.Name}\r\n" + Biz.GetErrorsAsString());
+                        u.Password = u.Login;
-                    throw new System.Exception("Seeder::GenSeedUser error creating user\r\n" + Biz.GetErrorsAsString());
+                    u.Roles = roles;
+                    u.LocaleId = localeId == 0 ? ServerBootConfig.AYANOVA_DEFAULT_LANGUAGE_ID : localeId;
+                    u.UserType = userType;
+                    u.EmployeeNumber = "A-" + (454 + SeededUserCount).ToString() + "-Y";
+                    u.Notes = Fake.Lorem.Sentence();//Fake.Lorem.Paragraph(2);
+                    //TODO: After have USER and HEADOFFICE and VENDOR, if usertype is subcontractor or client or headoffice it needs to set a corresponding user's parent org record id to go with it
+                    u.Tags = RandomTags(Fake);
+                    //Children and relations
+                    u.UserOptions = new UserOptions();
+                    u.UserOptions.EmailAddress = p.Email.Replace("gmail.com", "helloayanova.com").Replace("hotmail.com", "helloayanova.com").Replace("yahoo.com", "helloayanova.com");
+                    u.UserOptions.TimeZoneOffset = timeZoneOffset;
+
+                    var NewObject = Biz.Create(cct, u);
+                    if (NewObject == null)
+                    {
+                        log.LogError($"Seeder::GenSeedUser error creating user {u.Name}\r\n" + Biz.GetErrorsAsString());
+                        throw new System.Exception("Seeder::GenSeedUser error creating user\r\n" + Biz.GetErrorsAsString());
+                    }
                 }
             }
 
@@ -587,8 +566,24 @@ namespace AyaNova.Util
         //////////////////////////////////////////////////////
         //Seed widget for testing
         //
-        public static async Task GenSeedWidgetAsync(ILogger log, int count)
+        public static void GenSeedWidget(ILogger log, int count)
         {
+            //COMMIT BEFORE GOING NON ASYNC AGAIN TO SEE IF MEDIUM 500 WIDGETS STILL TAKES SO LONG: 
+            //ASYNC CACHED STUFF 2020-01-23 15:58:31.1619|INFO|Seeder|500 Widgets seeded in 54890 ms
+            //NON ASYNC ORIGINAL METHOD 2020-01-23 16:19:59.9131|INFO|Seeder|500 Widgets seeded in 51442 ms
+            //NON ASYNC HERE BUT CALLING ASYNC CREATE IN WIDGET: 2020-01-23 16:23:58.9478|INFO|Seeder|500 Widgets seeded in 53362 ms
+            //NOT CALLING SAVE ON WIDGET BUT DOING ALL OTHER CODE: 2020-01-23 16:25:45.0293|INFO|Seeder|500 Widgets seeded in 30 ms
+            //So slowness is all in the save code, let's see whats what there...
+            // first tried not caching teh widgetbiz object: 2020-01-23 16:28:58.0876|INFO|Seeder|500 Widgets seeded in 53577 ms
+            //so that's pointless to cache it
+            //ONE LAST THING IN HERE, trying *NOT* caching the db context and create each time: 2020-01-23 16:32:13.8966|INFO|Seeder|500 Widgets seeded in 22679 ms
+            //  - Holy shit, that's back to where it was a few days ago, so for some reason using the same context over and over is slowing shit down drastically
+            //  ripping that out now.
+
+            //OK, making changes in CREATE code in widgetbiz
+            //First removed extra save call and async db access shaved off a second
+            //the commented out the Search indexer and it dropped from 22 seconds to 4.5!!!!
+            //So the search indexing code is super fucking slow 2020-01-23 16:45:20.9518|INFO|Seeder|500 Widgets seeded in 4572 ms
 
             var f = new Bogus.Faker();
 
@@ -608,14 +603,15 @@ namespace AyaNova.Util
                 //Random but valid enum
                 AuthorizationRoles randomRole = (AuthorizationRoles)values.GetValue(random.Next(values.Length));
                 o.Roles = randomRole;
-                o.Notes = f.Lorem.Paragraphs();
+               
+                o.Notes = f.Lorem.Sentence();
                 o.Tags = RandomTags(f);
 
                 o.UserId = f.Random.Int(1, SeededUserCount);
 
                 //RANDOM CUSTOM FIELD DATA              
                 var c1 = DateUtil.UniversalISO8661Format(f.Date.Between(DateTime.Now.AddYears(-1), DateTime.Now.AddYears(1)));
-                var c2 = f.Lorem.Paragraph();
+                var c2 = f.Lorem.Sentence();
                 var c3 = f.Random.Int(1, 99999999);
                 var c4 = f.Random.Bool().ToString().ToLowerInvariant();
                 var c5 = f.Random.Decimal();
@@ -623,13 +619,16 @@ namespace AyaNova.Util
                 o.CustomFields = $@"{{c1:""{c1}"",c2:""{c2}"",c3:{c3},c4:{c4},c5:{c5}}}";
 
 
-                var NewObject = await Cached_WidgetBiz.CreateAsync(o);
+                //var NewObject = Cached_WidgetBiz.CreateAsync(o).Result;
+                //test without cached widgetbiz
+                WidgetBiz biz = WidgetBiz.GetBizInternal(ServiceProviderProvider.DBContext);
+                var NewObject = biz.CreateAsync(o).Result;
                 if (NewObject == null)
-                {                    
+                {
-                    log.LogError($"Seeder::GenSeedWidget error creating widget {o.Name}\r\n" + Cached_WidgetBiz.GetErrorsAsString());
+                    log.LogError($"Seeder::GenSeedWidget error creating widget {o.Name}\r\n" + biz.GetErrorsAsString());
-                    throw new System.Exception("Seeder::GenSeedWidget error creating widget\r\n" + Cached_WidgetBiz.GetErrorsAsString());
+                    throw new System.Exception("Seeder::GenSeedWidget error creating widget\r\n" + biz.GetErrorsAsString());
                 }
-                
+
             }
         }