From 2bd5831a38cfd0cd8dd04a712404a4a0f566a229 Mon Sep 17 00:00:00 2001 From: John Cardinal Date: Thu, 11 Oct 2018 17:14:15 +0000 Subject: [PATCH] --- devdocs/todo.txt | 4 ++++ .../Controllers/AttachmentController.cs | 4 ++-- server/AyaNova/Program.cs | 5 +++++ server/AyaNova/Startup.cs | 2 +- server/AyaNova/biz/Search.cs | 16 ++++++++------ server/AyaNova/biz/TagBiz.cs | 22 +++++++++---------- server/AyaNova/biz/TagGroupBiz.cs | 8 +++---- server/AyaNova/biz/UserBiz.cs | 10 ++++----- server/AyaNova/biz/WidgetBiz.cs | 10 ++++----- 9 files changed, 46 insertions(+), 35 deletions(-) diff --git a/devdocs/todo.txt b/devdocs/todo.txt index 89d426f5..ec946be7 100644 --- a/devdocs/todo.txt +++ b/devdocs/todo.txt @@ -28,10 +28,14 @@ Once that is done then can steam ahead on the biz objects but until I have the c IMMEDIATE ITEMS: ================ +- fixup removal of db context from search processing, then... - Re-test search with new processing code, do a huge test locally repeatedly for a bit - If ok then move on to do the devops test again with a fresh build and huge db +- Whole point is to eliminate the exceptions and have it work properly. +- Also maybe find better way using lock() code to generate unique ID numbers for the unit tests + - Put up copy to devops, build huge dataset, run all tests, pound it with 1000 runs, shake it out - Starting 1000 test runs at 2018-10-9-4:12pm diff --git a/server/AyaNova/Controllers/AttachmentController.cs b/server/AyaNova/Controllers/AttachmentController.cs index 7cfe8c4d..c527d314 100644 --- a/server/AyaNova/Controllers/AttachmentController.cs +++ b/server/AyaNova/Controllers/AttachmentController.cs @@ -233,7 +233,7 @@ namespace AyaNova.Api.Controllers EventLogProcessor.LogEventToDatabase(new Event(UserId, attachToObject.ObjectId, attachToObject.ObjectType, AyaEvent.AttachmentCreate, v.DisplayFileName), ct); //SEARCH INDEXING - Search.ProcessNewObjectKeywords(ct, UserLocaleIdFromContext.Id(HttpContext.Items), v.Id, AyaType.FileAttachment, v.DisplayFileName, v.DisplayFileName, v.Notes, v.StoredFileName); + Search.ProcessNewObjectKeywords( UserLocaleIdFromContext.Id(HttpContext.Items), v.Id, AyaType.FileAttachment, v.DisplayFileName, v.DisplayFileName, v.Notes, v.StoredFileName); } } @@ -303,7 +303,7 @@ namespace AyaNova.Api.Controllers EventLogProcessor.LogEventToDatabase(new Event(UserId, dbObj.AttachToObjectId, dbObj.AttachToObjectType, AyaEvent.AttachmentDelete, dbObj.DisplayFileName), ct); //Delete search index - Search.ProcessDeletedObjectKeywords(ct, dbObj.Id, AyaType.FileAttachment); + Search.ProcessDeletedObjectKeywords(dbObj.Id, AyaType.FileAttachment); return NoContent(); } diff --git a/server/AyaNova/Program.cs b/server/AyaNova/Program.cs index 6a1a5965..3bc754df 100644 --- a/server/AyaNova/Program.cs +++ b/server/AyaNova/Program.cs @@ -104,6 +104,10 @@ namespace AyaNova var logRuleFilterOutMicrosoftEfCoreCommandExceptions = new LoggingRule("Microsoft.EntityFrameworkCore.Database.Command", NLog.LogLevel.Trace, NLog.LogLevel.Error, nullTarget); logRuleFilterOutMicrosoftEfCoreCommandExceptions.Final = true; + var logRuleFilterOutMicrosoftEfCoreDbUpdateExceptions = new LoggingRule("Microsoft.EntityFrameworkCore.DbUpdateException", NLog.LogLevel.Trace, NLog.LogLevel.Error, nullTarget); + logRuleFilterOutMicrosoftEfCoreDbUpdateExceptions.Final = true; + + //Log all other regular items at selected level var logRuleAyaNovaItems = new LoggingRule("*", NLogLevel, fileTarget); @@ -121,6 +125,7 @@ namespace AyaNova logConfig.LoggingRules.Add(logRuleFilterOutMicrosoft); logConfig.LoggingRules.Add(logRuleFilterOutMicrosoftEfCoreConcurrencyExceptions); logConfig.LoggingRules.Add(logRuleFilterOutMicrosoftEfCoreCommandExceptions); + logConfig.LoggingRules.Add(logRuleFilterOutMicrosoftEfCoreDbUpdateExceptions); } logConfig.LoggingRules.Add(logRuleAyaNovaItems); diff --git a/server/AyaNova/Startup.cs b/server/AyaNova/Startup.cs index 5e2b5adc..5213dcfd 100644 --- a/server/AyaNova/Startup.cs +++ b/server/AyaNova/Startup.cs @@ -105,7 +105,7 @@ namespace AyaNova services.AddEntityFrameworkNpgsql().AddDbContext( options => options.UseNpgsql(_connectionString - //,opt => opt.EnableRetryOnFailure() + //,opt => opt.EnableRetryOnFailure()//REMOVED THIS BECAUSE IT WAS INTEFERING WITH TRANSACTIONS BUT THEN DIDN'T USE THE TRANSACTION BUT IT SEEMS FASTER WITHOUT IT AS WELL SO...?? )//http://www.npgsql.org/efcore/misc.html?q=execution%20strategy#execution-strategy .ConfigureWarnings(warnings => //https://livebook.manning.com/#!/book/entity-framework-core-in-action/chapter-12/v-10/85 warnings.Throw( //Throw an exception on client eval, not necessarily an error but a smell diff --git a/server/AyaNova/biz/Search.cs b/server/AyaNova/biz/Search.cs index 5a2d2405..281c75a4 100644 --- a/server/AyaNova/biz/Search.cs +++ b/server/AyaNova/biz/Search.cs @@ -375,20 +375,21 @@ namespace AyaNova.Biz #region ProcessKeywords into Database - public static void ProcessNewObjectKeywords(AyContext ct, long localeId, long objectID, AyaType objectType, string name, params string[] text) + public static void ProcessNewObjectKeywords(long localeId, long objectID, AyaType objectType, string name, params string[] text) { - ProcessKeywords(ct, localeId, objectID, objectType, true, name, text); + ProcessKeywords(localeId, objectID, objectType, true, name, text); } - public static void ProcessUpdatedObjectKeywords(AyContext ct, long localeId, long objectID, AyaType objectType, string name, params string[] text) + public static void ProcessUpdatedObjectKeywords(long localeId, long objectID, AyaType objectType, string name, params string[] text) { - ProcessKeywords(ct, localeId, objectID, objectType, false, name, text); + ProcessKeywords(localeId, objectID, objectType, false, name, text); } - public static void ProcessDeletedObjectKeywords(AyContext ct, long objectID, AyaType objectType) + public static void ProcessDeletedObjectKeywords(long objectID, AyaType objectType) { //Be careful in future, if you put ToString at the end of each object in the string interpolation //npgsql driver will assume it's a string and put quotes around it triggering an error that a string can't be compared to an int + AyContext ct = ServiceProviderProvider.DBContext; ct.Database.ExecuteSqlCommand($"delete from asearchkey where objectid={objectID} and objecttype={(int)objectType}"); } @@ -397,9 +398,10 @@ namespace AyaNova.Biz /// Process the keywords into the dictionary /// NOTE: NAME parameter is in ADDITION to the NAME also being on of the strings passed in text parameter /// - private static void ProcessKeywords(AyContext ct, long localeId, long objectID, AyaType objectType, bool newRecord, string name, params string[] text) + private static void ProcessKeywords(long localeId, long objectID, AyaType objectType, bool newRecord, string name, params string[] text) { + AyContext ct = ServiceProviderProvider.DBContext; #if (DEBUG) if (objectType == AyaType.TagMap || objectType == AyaType.JobOperations || objectType == AyaType.Locale) { @@ -411,7 +413,7 @@ namespace AyaNova.Biz //IF NOT NEW, DELETE ALL EXISTING ENTRIES FOR OBJECT TYPE AND ID if (!newRecord) { - ProcessDeletedObjectKeywords(ct, objectID, objectType); + ProcessDeletedObjectKeywords(objectID, objectType); } //BREAK STRING ARRAY INTO KEYWORD LIST diff --git a/server/AyaNova/biz/TagBiz.cs b/server/AyaNova/biz/TagBiz.cs index df7f3fec..5bbf3549 100644 --- a/server/AyaNova/biz/TagBiz.cs +++ b/server/AyaNova/biz/TagBiz.cs @@ -48,8 +48,8 @@ namespace AyaNova.Biz { //This is simple so nothing more here, but often will be copying to a different output object or some other ops return await ct.Tag.SingleOrDefaultAsync(m => m.Id == fetchId); - } - + } + //////////////////////////////////////////////////////////////////////////////////////////////// //CREATE @@ -76,10 +76,10 @@ namespace AyaNova.Biz //Handle child and associated items: //EVENT LOG - EventLogProcessor.LogEventToDatabase(new Event(UserId, outObj.Id, BizType, AyaEvent.Created), ct); + EventLogProcessor.LogEventToDatabase(new Event(UserId, outObj.Id, BizType, AyaEvent.Created), ct); //SEARCH INDEXING - Search.ProcessNewObjectKeywords(ct, UserLocaleId, outObj.Id, BizType, outObj.Name, outObj.Name); + Search.ProcessNewObjectKeywords(UserLocaleId, outObj.Id, BizType, outObj.Name, outObj.Name); return outObj; } @@ -101,7 +101,7 @@ namespace AyaNova.Biz return inObj; } - + @@ -185,9 +185,9 @@ namespace AyaNova.Biz //Log modification EventLogProcessor.LogEventToDatabase(new Event(UserId, dbObj.Id, BizType, AyaEvent.Modified), ct); - + //Update keywords - Search.ProcessUpdatedObjectKeywords(ct, UserLocaleId, dbObj.Id, BizType, dbObj.Name, dbObj.Name); + Search.ProcessUpdatedObjectKeywords( UserLocaleId, dbObj.Id, BizType, dbObj.Name, dbObj.Name); return true; @@ -210,9 +210,9 @@ namespace AyaNova.Biz //Log modification EventLogProcessor.LogEventToDatabase(new Event(UserId, dbObj.Id, BizType, AyaEvent.Modified), ct); - + //Update keywords - Search.ProcessUpdatedObjectKeywords(ct, UserLocaleId, dbObj.Id, BizType, dbObj.Name, dbObj.Name); + Search.ProcessUpdatedObjectKeywords( UserLocaleId, dbObj.Id, BizType, dbObj.Name, dbObj.Name); return true; @@ -240,7 +240,7 @@ namespace AyaNova.Biz ct.SaveChanges(); //Delete search index - Search.ProcessDeletedObjectKeywords(ct, dbObj.Id, BizType); + Search.ProcessDeletedObjectKeywords(dbObj.Id, BizType); return true; } @@ -258,7 +258,7 @@ namespace AyaNova.Biz ct.Database.ExecuteSqlCommand($"delete from ataggroupmap where tagid = {dbObj.Id}"); //Log EventLogProcessor.LogEventToDatabase(new Event(UserId, dbObj.Id, BizType, AyaEvent.TagMassUntag), ct); - + return true; } diff --git a/server/AyaNova/biz/TagGroupBiz.cs b/server/AyaNova/biz/TagGroupBiz.cs index 33d2718b..e0092ce6 100644 --- a/server/AyaNova/biz/TagGroupBiz.cs +++ b/server/AyaNova/biz/TagGroupBiz.cs @@ -69,7 +69,7 @@ namespace AyaNova.Biz EventLogProcessor.LogEventToDatabase(new Event(UserId, outObj.Id, BizType, AyaEvent.Created), ct); //SEARCH INDEXING - Search.ProcessNewObjectKeywords(ct, UserLocaleId, outObj.Id, BizType, outObj.Name, outObj.Name); + Search.ProcessNewObjectKeywords( UserLocaleId, outObj.Id, BizType, outObj.Name, outObj.Name); return outObj; } @@ -204,7 +204,7 @@ namespace AyaNova.Biz EventLogProcessor.LogEventToDatabase(new Event(UserId, dbObj.Id, BizType, AyaEvent.Modified), ct); //Update keywords - Search.ProcessUpdatedObjectKeywords(ct, UserLocaleId, dbObj.Id, BizType, dbObj.Name, dbObj.Name); + Search.ProcessUpdatedObjectKeywords( UserLocaleId, dbObj.Id, BizType, dbObj.Name, dbObj.Name); return true; } @@ -228,7 +228,7 @@ namespace AyaNova.Biz EventLogProcessor.LogEventToDatabase(new Event(UserId, dbObj.Id, BizType, AyaEvent.Modified), ct); //Update keywords - Search.ProcessUpdatedObjectKeywords(ct, UserLocaleId, dbObj.Id, BizType, dbObj.Name, dbObj.Name); + Search.ProcessUpdatedObjectKeywords( UserLocaleId, dbObj.Id, BizType, dbObj.Name, dbObj.Name); return true; } @@ -253,7 +253,7 @@ namespace AyaNova.Biz ct.SaveChanges(); //Delete search index - Search.ProcessDeletedObjectKeywords(ct, dbObj.Id, BizType); + Search.ProcessDeletedObjectKeywords(dbObj.Id, BizType); return true; } diff --git a/server/AyaNova/biz/UserBiz.cs b/server/AyaNova/biz/UserBiz.cs index 7275f374..8df75b45 100644 --- a/server/AyaNova/biz/UserBiz.cs +++ b/server/AyaNova/biz/UserBiz.cs @@ -75,7 +75,7 @@ namespace AyaNova.Biz EventLogProcessor.LogEventToDatabase(new Event(UserId, outObj.Id, BizType, AyaEvent.Created), ct); //SEARCH INDEXING - Search.ProcessNewObjectKeywords(ct, UserLocaleId, outObj.Id, BizType, outObj.Name, outObj.EmployeeNumber, outObj.Notes, outObj.Name); + Search.ProcessNewObjectKeywords( UserLocaleId, outObj.Id, BizType, outObj.Name, outObj.EmployeeNumber, outObj.Notes, outObj.Name); return outObj; @@ -112,7 +112,7 @@ namespace AyaNova.Biz EventLogProcessor.LogEventToDatabase(new Event(UserId, outObj.Id, BizType, AyaEvent.Created), TempContext); //SEARCH INDEXING - Search.ProcessNewObjectKeywords(TempContext, UserLocaleId, outObj.Id, BizType, outObj.Name, outObj.EmployeeNumber, outObj.Notes, outObj.Name); + Search.ProcessNewObjectKeywords(UserLocaleId, outObj.Id, BizType, outObj.Name, outObj.EmployeeNumber, outObj.Notes, outObj.Name); return outObj; @@ -253,7 +253,7 @@ namespace AyaNova.Biz //Log modification EventLogProcessor.LogEventToDatabase(new Event(UserId, dbObj.Id, BizType, AyaEvent.Modified), ct); //Update keywords - Search.ProcessUpdatedObjectKeywords(ct, UserLocaleId, dbObj.Id, BizType, dbObj.Name, dbObj.EmployeeNumber, dbObj.Notes, dbObj.Name); + Search.ProcessUpdatedObjectKeywords( UserLocaleId, dbObj.Id, BizType, dbObj.Name, dbObj.EmployeeNumber, dbObj.Notes, dbObj.Name); return true; } @@ -287,7 +287,7 @@ namespace AyaNova.Biz EventLogProcessor.LogEventToDatabase(new Event(UserId, dbObj.Id, BizType, AyaEvent.Modified), ct); //Update keywords - Search.ProcessUpdatedObjectKeywords(ct, UserLocaleId, dbObj.Id, BizType, dbObj.Name, dbObj.EmployeeNumber, dbObj.Notes, dbObj.Name); + Search.ProcessUpdatedObjectKeywords( UserLocaleId, dbObj.Id, BizType, dbObj.Name, dbObj.EmployeeNumber, dbObj.Notes, dbObj.Name); return true; } @@ -315,7 +315,7 @@ namespace AyaNova.Biz ct.SaveChanges(); //Delete search index - Search.ProcessDeletedObjectKeywords(ct, dbObj.Id, BizType); + Search.ProcessDeletedObjectKeywords(dbObj.Id, BizType); //TAGS TagMapBiz.DeleteAllForObject(new AyaTypeId(BizType, dbObj.Id), ct); diff --git a/server/AyaNova/biz/WidgetBiz.cs b/server/AyaNova/biz/WidgetBiz.cs index 8767955e..fe29a0d6 100644 --- a/server/AyaNova/biz/WidgetBiz.cs +++ b/server/AyaNova/biz/WidgetBiz.cs @@ -78,7 +78,7 @@ namespace AyaNova.Biz EventLogProcessor.LogEventToDatabase(new Event(UserId, outObj.Id, BizType, AyaEvent.Created), ct); //SEARCH INDEXING - Search.ProcessNewObjectKeywords(ct, UserLocaleId, outObj.Id, BizType, outObj.Name, outObj.Notes, outObj.Name, outObj.Serial.ToString()); + Search.ProcessNewObjectKeywords( UserLocaleId, outObj.Id, BizType, outObj.Name, outObj.Notes, outObj.Name, outObj.Serial.ToString()); return outObj; @@ -109,7 +109,7 @@ namespace AyaNova.Biz EventLogProcessor.LogEventToDatabase(new Event(UserId, outObj.Id, BizType, AyaEvent.Created), TempContext); //SEARCH INDEXING - Search.ProcessNewObjectKeywords(TempContext, UserLocaleId, outObj.Id, BizType, outObj.Name, outObj.Notes, outObj.Name, outObj.Serial.ToString()); + Search.ProcessNewObjectKeywords(UserLocaleId, outObj.Id, BizType, outObj.Name, outObj.Notes, outObj.Name, outObj.Serial.ToString()); return outObj; @@ -225,7 +225,7 @@ namespace AyaNova.Biz EventLogProcessor.LogEventToDatabase(new Event(UserId, dbObj.Id, BizType, AyaEvent.Modified), ct); //Update keywords - Search.ProcessUpdatedObjectKeywords(ct, UserLocaleId, dbObj.Id, BizType, dbObj.Name, dbObj.Notes, dbObj.Name, dbObj.Serial.ToString()); + Search.ProcessUpdatedObjectKeywords( UserLocaleId, dbObj.Id, BizType, dbObj.Name, dbObj.Notes, dbObj.Name, dbObj.Serial.ToString()); return true; } @@ -248,7 +248,7 @@ namespace AyaNova.Biz EventLogProcessor.LogEventToDatabase(new Event(UserId, dbObj.Id, BizType, AyaEvent.Modified), ct); //Update keywords - Search.ProcessUpdatedObjectKeywords(ct, UserLocaleId, dbObj.Id, BizType, dbObj.Name, dbObj.Notes, dbObj.Name, dbObj.Serial.ToString()); + Search.ProcessUpdatedObjectKeywords( UserLocaleId, dbObj.Id, BizType, dbObj.Name, dbObj.Notes, dbObj.Name, dbObj.Serial.ToString()); return true; } @@ -275,7 +275,7 @@ namespace AyaNova.Biz ct.SaveChanges(); //Delete search index - Search.ProcessDeletedObjectKeywords(ct, dbObj.Id, BizType); + Search.ProcessDeletedObjectKeywords(dbObj.Id, BizType); //TAGS TagMapBiz.DeleteAllForObject(new AyaTypeId(BizType, dbObj.Id), ct);