From 90052f100027ef27649dcdc50213ab663c3b19d5 Mon Sep 17 00:00:00 2001 From: John Cardinal Date: Wed, 29 Jul 2020 23:13:20 +0000 Subject: [PATCH] --- .vscode/launch.json | 4 ++-- server/AyaNova/Program.cs | 11 +++++----- server/AyaNova/biz/JobsBiz.cs | 18 +++++++++------- server/AyaNova/biz/UserBiz.cs | 6 ++++-- server/AyaNova/generator/CoreJobNotify.cs | 2 +- server/AyaNova/generator/CoreJobSweeper.cs | 25 +++++++++++++++------- server/AyaNova/generator/Generate.cs | 2 +- 7 files changed, 40 insertions(+), 28 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index fe194dde..9efe9077 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -42,8 +42,8 @@ "AYANOVA_JWT_SECRET": "UNLICENSED5G*QQJ8#bQ7$Xr_@sXfHq4", //"AYANOVA_SET_SUPERUSER_PW": "l3tm3in", "AYANOVA_LOG_LEVEL": "Info", - //"AYANOVA_LOG_LEVEL": "Debug", - //"AYANOVA_LOG_LEVEL": "Trace", + // "AYANOVA_LOG_LEVEL": "Debug", + // "AYANOVA_LOG_LEVEL": "Trace", "AYANOVA_DEFAULT_TRANSLATION": "en", //TRANSLATION MUST BE en for Integration TESTING //"AYANOVA_PERMANENTLY_ERASE_DATABASE": "true", diff --git a/server/AyaNova/Program.cs b/server/AyaNova/Program.cs index 65e19e1d..bf5cb404 100644 --- a/server/AyaNova/Program.cs +++ b/server/AyaNova/Program.cs @@ -23,7 +23,7 @@ namespace AyaNova //Boot lock for generator ServerGlobalOpsSettingsCache.BOOTING = true; - + //Get config var config = new ConfigurationBuilder().AddEnvironmentVariables().AddCommandLine(args).Build(); ServerBootConfig.SetConfiguration(config); @@ -39,7 +39,7 @@ namespace AyaNova //default log level NLog.LogLevel NLogLevel = NLog.LogLevel.Info; - bool logLevelIsInfoOrHigher = true; + bool FilterOutMicrosoftLogItems = true; switch (ServerBootConfig.AYANOVA_LOG_LEVEL.ToLowerInvariant()) { @@ -57,11 +57,10 @@ namespace AyaNova break; case "debug": NLogLevel = NLog.LogLevel.Debug; - logLevelIsInfoOrHigher = false; break; case "trace": NLogLevel = NLog.LogLevel.Trace; - logLevelIsInfoOrHigher = false; + FilterOutMicrosoftLogItems = false;//Only at TRACE level do we want to see all the Microsoft logging stuff break; default: NLogLevel = NLog.LogLevel.Info; @@ -134,7 +133,7 @@ namespace AyaNova logConfig.LoggingRules.Add(logRuleForConsole); //only log microsoft stuff it log is debug level or lower - if (logLevelIsInfoOrHigher) + if (FilterOutMicrosoftLogItems) { //filter OUT microsoft stuff logConfig.LoggingRules.Add(logRuleFilterOutMicrosoft); @@ -165,7 +164,7 @@ namespace AyaNova //log configuration try { - var AyaNovaConfig = config.AsEnumerable().Where(z => z.Key.StartsWith("AYANOVA") && z.Key != "AYANOVA_JWT_SECRET"&& z.Key != "AYANOVA_SET_SUPERUSER_PW").Select(z => z.Key + "=" + z.Value).ToList(); + var AyaNovaConfig = config.AsEnumerable().Where(z => z.Key.StartsWith("AYANOVA") && z.Key != "AYANOVA_JWT_SECRET" && z.Key != "AYANOVA_SET_SUPERUSER_PW").Select(z => z.Key + "=" + z.Value).ToList(); var DiagConfig = string.Join(",", AyaNovaConfig); DiagConfig = DbUtil.PasswordRedactedConnectionString(DiagConfig); logger.Info($"Config {DiagConfig}"); diff --git a/server/AyaNova/biz/JobsBiz.cs b/server/AyaNova/biz/JobsBiz.cs index abeac880..6be91b27 100644 --- a/server/AyaNova/biz/JobsBiz.cs +++ b/server/AyaNova/biz/JobsBiz.cs @@ -182,14 +182,16 @@ namespace AyaNova.Biz //system lock (no license) is a complete deal breaker for continuation beyond here if (serverState.IsSystemLocked) return; + //Moved the following user count checker to within JOB SWEEPER so that it only checks on every half hour so it's less obvious in logs with full trace on and + //also so someone fucking with the system won't get locked out right away. //"SHENANIGAN" CHECK, note, must be here after systemlock because it relies on there being a license - if (await UserBiz.ActiveCountAsync() > AyaNova.Core.License.ActiveKey.ActiveNumber) - { - var msg = $"E1020 - Active count exceeded capacity"; - ServiceProviderProvider.ServerState.SetSystemLock(msg); - log.LogCritical(msg); - return; - } + // if (await UserBiz.ActiveCountAsync() > AyaNova.Core.License.ActiveKey.ActiveNumber) + // { + // var msg = $"E1020 - Active count exceeded capacity"; + // ServiceProviderProvider.ServerState.SetSystemLock(msg); + // log.LogCritical(msg); + // return; + // } //BACKUP @@ -199,7 +201,7 @@ namespace AyaNova.Biz await CoreJobNotify.DoWorkAsync(); await CoreNotificationSweeper.DoWorkAsync(); - //JOB SWEEPER + //JOB SWEEPER / AND USER COUNT CHECK await CoreJobSweeper.DoWorkAsync(); //### API Open only jobs diff --git a/server/AyaNova/biz/UserBiz.cs b/server/AyaNova/biz/UserBiz.cs index cd5bd13a..0e697cf8 100644 --- a/server/AyaNova/biz/UserBiz.cs +++ b/server/AyaNova/biz/UserBiz.cs @@ -34,6 +34,8 @@ namespace AyaNova.Biz { using (AyContext ct = ServiceProviderProvider.DBContext) { + // var ret = await ct.Database.ExecuteSqlRawAsync($"SELECT COUNT(*) FROM auser AS a WHERE (a.active = TRUE) AND ((a.usertype = 2) OR (a.usertype = 7))"); + var ret = await ct.User.AsNoTracking().Where(z => z.Active == true && ( z.UserType == UserType.Service || z.UserType == UserType.ServiceContractor)).LongCountAsync(); @@ -299,8 +301,8 @@ namespace AyaNova.Biz await SearchIndexAsync(dbObject, false); await TagBiz.ProcessUpdateTagsInRepositoryAsync(ct, dbObject.Tags, SnapshotOfOriginalDBObj.Tags); await NotifyEventProcessor.HandlePotentialNotificationEvent(AyaEvent.Modified, dbObject, SnapshotOfOriginalDBObj); - - + + return dbObject; } diff --git a/server/AyaNova/generator/CoreJobNotify.cs b/server/AyaNova/generator/CoreJobNotify.cs index aff81644..d6893d4d 100644 --- a/server/AyaNova/generator/CoreJobNotify.cs +++ b/server/AyaNova/generator/CoreJobNotify.cs @@ -107,7 +107,7 @@ namespace AyaNova.Biz //All items have an event date, for non time delayed events it's just the moment it was created //which will predate this moment now if it's pre-existing var events = await ct.NotifyEvent.Include(z => z.NotifySubscription).ToListAsync(); - log.LogDebug($"Found {events.Count} NotifyEvents to examine for potential delivery"); + log.LogTrace($"Found {events.Count} NotifyEvents to examine for potential delivery"); //cache translations //Get all subscription unique userId's that aren't inapp deliveries diff --git a/server/AyaNova/generator/CoreJobSweeper.cs b/server/AyaNova/generator/CoreJobSweeper.cs index 7d74bbb8..c9562970 100644 --- a/server/AyaNova/generator/CoreJobSweeper.cs +++ b/server/AyaNova/generator/CoreJobSweeper.cs @@ -17,7 +17,7 @@ namespace AyaNova.Biz { private static ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("CoreJobSweeper"); private static DateTime lastSweep = DateTime.MinValue; - private static TimeSpan SWEEP_EVERY_INTERVAL = new TimeSpan(0, 30, 0); + private static TimeSpan SWEEP_EVERY_INTERVAL = new TimeSpan(0, 0, 10); private static TimeSpan SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN = new TimeSpan(14, 0, 0, 0);//14 days private static TimeSpan FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN = new TimeSpan(14, 0, 0, 0);//14 days (gives people time to notice and look into it) private static TimeSpan INTERNAL_JOBS_LOGS_DELETE_AFTER_THIS_TIMESPAN = new TimeSpan(14, 0, 0, 0);//14 days @@ -28,7 +28,7 @@ namespace AyaNova.Biz // DoSweep // public static async Task DoWorkAsync() - { + { //This will get triggered roughly every minute, but we don't want to sweep that frequently if (DateTime.UtcNow - lastSweep < SWEEP_EVERY_INTERVAL) return; @@ -40,22 +40,31 @@ namespace AyaNova.Biz //calculate cutoff to delete DateTime dtDeleteCutoff = DateTime.UtcNow - SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN; await sweepAsync(ct, dtDeleteCutoff, JobStatus.Completed); - + //SWEEP FAILED JOBS //calculate cutoff to delete dtDeleteCutoff = DateTime.UtcNow - FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN; await sweepAsync(ct, dtDeleteCutoff, JobStatus.Failed); - + //KILL STUCK JOBS //calculate cutoff to delete DateTime dtRunningDeadline = DateTime.UtcNow - RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN; await killStuckJobsAsync(ct, dtRunningDeadline); - + //SWEEP INTERNAL JOB LOG //calculate cutoff to delete dtDeleteCutoff = DateTime.UtcNow - INTERNAL_JOBS_LOGS_DELETE_AFTER_THIS_TIMESPAN; await SweepInternalJobsLogsAsync(ct, dtDeleteCutoff); - + + //Stealthy check of user count exceeded + if (await UserBiz.ActiveCountAsync() > AyaNova.Core.License.ActiveKey.ActiveNumber) + { + var msg = $"E1020 - Active count exceeded capacity"; + AyaNova.Util.ServiceProviderProvider.ServerState.SetSystemLock(msg); + log.LogCritical(msg); + return; + } + } lastSweep = DateTime.UtcNow; } @@ -76,7 +85,7 @@ namespace AyaNova.Biz { try { - + await JobsBiz.RemoveJobAndLogsAsync(j.GId); } catch (Exception ex) @@ -114,7 +123,7 @@ namespace AyaNova.Biz private static async Task SweepInternalJobsLogsAsync(AyContext ct, DateTime dtDeleteCutoff) - { + { //Get the deleteable list (this is for reporting, could easily just do it in one go) var logs = await ct.OpsJobLog .AsNoTracking() diff --git a/server/AyaNova/generator/Generate.cs b/server/AyaNova/generator/Generate.cs index 8a70961a..78c6cc0c 100644 --- a/server/AyaNova/generator/Generate.cs +++ b/server/AyaNova/generator/Generate.cs @@ -53,7 +53,7 @@ namespace AyaNova.Generator { if (!ServerGlobalOpsSettingsCache.BOOTING) { - log.LogDebug($"GeneratorService running jobs"); + log.LogTrace($"GeneratorService running jobs"); //================================================================= try