From 126ae54f6605aec1f9cc5ea93903977cf73ffce9 Mon Sep 17 00:00:00 2001 From: John Cardinal Date: Wed, 18 Jan 2023 03:12:39 +0000 Subject: [PATCH] --- server/biz/NotifyEventType.cs | 2 +- server/biz/SubscriptionServerBiz.cs | 146 +++++++++--------- .../SockBotSubscriptionServerHealthChecks.cs | 91 ++--------- server/util/Mailer.cs | 2 +- 4 files changed, 93 insertions(+), 148 deletions(-) diff --git a/server/biz/NotifyEventType.cs b/server/biz/NotifyEventType.cs index 725edb4..fd72cbb 100644 --- a/server/biz/NotifyEventType.cs +++ b/server/biz/NotifyEventType.cs @@ -34,7 +34,7 @@ Inspiring quotes used to help complete this huge project by myself ServerStateStatusChange = 36, ServerStateStatusAge=37, SubscriptionServerExpiring = 38, - SubscriptionServerLastUpdateAge = 39, + SubscriptionServerLastUpdateAge = 39,//here this means the operating system, not RAVEN which should always be updated on all servers when released SubscriptionServerRequestReceived = 40, LicenseTrialRequestReceived = 41, PurchaseNotificationReceived = 42 diff --git a/server/biz/SubscriptionServerBiz.cs b/server/biz/SubscriptionServerBiz.cs index 8d512f8..8e03658 100644 --- a/server/biz/SubscriptionServerBiz.cs +++ b/server/biz/SubscriptionServerBiz.cs @@ -408,15 +408,15 @@ namespace Sockeye.Biz await NotifyEventHelper.ProcessStandardObjectEvents(ayaEvent, proposedObj, ct); //SPECIFIC EVENTS FOR THIS OBJECT - SubscriptionServer o = (SubscriptionServer)proposedObj; - + SubscriptionServer ProposedServer = (SubscriptionServer)proposedObj; + //## DELETED EVENTS //any event added below needs to be removed, so //just blanket remove any event for this object of eventtype that would be added below here //do it regardless any time there's an update and then //let this code below handle the refreshing addition that could have changes - await NotifyEventHelper.ClearPriorEventsForObject(ct, BizType, o.Id, NotifyEventType.SubscriptionServerExpiring); - await NotifyEventHelper.ClearPriorEventsForObject(ct, BizType, o.Id, NotifyEventType.SubscriptionServerLastUpdateAge); + await NotifyEventHelper.ClearPriorEventsForObject(ct, BizType, ProposedServer.Id, NotifyEventType.SubscriptionServerExpiring); + await NotifyEventHelper.ClearPriorEventsForObject(ct, BizType, ProposedServer.Id, NotifyEventType.SubscriptionServerLastUpdateAge); //## CREATED / MODIFIED EVENTS @@ -424,7 +424,7 @@ namespace Sockeye.Biz { //# NEW TRIAL SERVER REQUEST - if (ayaEvent == SockEvent.Created && o.Trial == true) + if (ayaEvent == SockEvent.Created && ProposedServer.Trial == true) { { //Conditions: must match specific status id value and also tags below @@ -436,16 +436,16 @@ namespace Sockeye.Biz if (!await UserBiz.UserIsActive(sub.UserId)) continue; //Tag match? (will be true if no sub tags so always safe to call this) - if (NotifyEventHelper.ObjectHasAllSubscriptionTags(o.Tags, sub.Tags)) + if (NotifyEventHelper.ObjectHasAllSubscriptionTags(ProposedServer.Tags, sub.Tags)) { NotifyEvent n = new NotifyEvent() { EventType = NotifyEventType.SubscriptionServerRequestReceived, UserId = sub.UserId, SockType = BizType, - ObjectId = o.Id, + ObjectId = ProposedServer.Id, NotifySubscriptionId = sub.Id, - Name = $"{o.TrialCompany} - trial server requested" + Name = $"{ProposedServer.TrialCompany} - trial server requested" }; await ct.NotifyEvent.AddAsync(n); log.LogDebug($"Adding NotifyEvent: [{n.ToString()}]"); @@ -466,7 +466,7 @@ namespace Sockeye.Biz if (!await UserBiz.UserIsActive(sub.UserId)) continue; //Tag match? (will be true if no sub tags so always safe to call this) - if (NotifyEventHelper.ObjectHasAllSubscriptionTags(o.Tags, sub.Tags)) + if (NotifyEventHelper.ObjectHasAllSubscriptionTags(ProposedServer.Tags, sub.Tags)) { NotifyEvent n = new NotifyEvent() @@ -474,10 +474,10 @@ namespace Sockeye.Biz EventType = NotifyEventType.SubscriptionServerExpiring, UserId = sub.UserId, SockType = BizType, - ObjectId = o.Id, + ObjectId = ProposedServer.Id, NotifySubscriptionId = sub.Id, - Name = o.Name, - EventDate = o.SubscriptionExpire + Name = ProposedServer.Name, + EventDate = ProposedServer.SubscriptionExpire }; await ct.NotifyEvent.AddAsync(n); log.LogDebug($"Adding NotifyEvent: [{n.ToString()}]"); @@ -488,9 +488,9 @@ namespace Sockeye.Biz }//subscription server expiry event - //# SUBSCRIPTION SERVER LAST UPDATE AGE + //# SUBSCRIPTION SERVER LAST UPDATE (O.S.) AGE { - if (o.LastUpdated != null) + if (ProposedServer.LastUpdated != null) { //notify users about warranty expiry (time delayed) var subs = await ct.NotifySubscription.Where(z => z.EventType == NotifyEventType.SubscriptionServerLastUpdateAge).ToListAsync(); @@ -500,17 +500,17 @@ namespace Sockeye.Biz if (!await UserBiz.UserIsActive(sub.UserId)) continue; //Tag match? (will be true if no sub tags so always safe to call this) - if (NotifyEventHelper.ObjectHasAllSubscriptionTags(o.Tags, sub.Tags)) + if (NotifyEventHelper.ObjectHasAllSubscriptionTags(ProposedServer.Tags, sub.Tags)) { NotifyEvent n = new NotifyEvent() { EventType = NotifyEventType.SubscriptionServerLastUpdateAge, UserId = sub.UserId, SockType = BizType, - ObjectId = o.Id, + ObjectId = ProposedServer.Id, NotifySubscriptionId = sub.Id, - Name = o.Name, - EventDate = (DateTime)o.LastUpdated + Name = ProposedServer.Name, + EventDate = (DateTime)ProposedServer.LastUpdated }; await ct.NotifyEvent.AddAsync(n); log.LogDebug($"Adding NotifyEvent: [{n.ToString()}]"); @@ -522,66 +522,70 @@ namespace Sockeye.Biz }//subscription server last updated event - - //# STATUS CHANGE (create new status) + //################################## + //ServerState changes events + //################################## + if (isNew || ((SubscriptionServer)currentObj).ServerState != ProposedServer.ServerState)//must have changed from a prior state { - //Conditions: must match specific status id value and also tags below - //delivery is immediate so no need to remove old ones of this kind - var subs = await ct.NotifySubscription.AsNoTracking().Where(z => z.EventType == NotifyEventType.ServerStateStatusChange && z.IdValue == (long)o.ServerState).ToListAsync(); - foreach (var sub in subs) + //# STATUS CHANGE (changed to this new status) { - //not for inactive users - if (!await UserBiz.UserIsActive(sub.UserId)) continue; - - //Tag match? (will be true if no sub tags so always safe to call this) - if (NotifyEventHelper.ObjectHasAllSubscriptionTags(o.Tags, sub.Tags)) + //Conditions: must match specific status id value and also tags below + //delivery is immediate so no need to remove old ones of this kind + var subs = await ct.NotifySubscription.AsNoTracking().Where(z => z.EventType == NotifyEventType.ServerStateStatusChange && z.IdValue == (long)ProposedServer.ServerState).ToListAsync(); + foreach (var sub in subs) { - NotifyEvent n = new NotifyEvent() - { - EventType = NotifyEventType.ServerStateStatusChange, - UserId = sub.UserId, - SockType = BizType, - ObjectId = o.Id, - NotifySubscriptionId = sub.Id, - Name = $"{o.Name} - {o.ServerState.ToString()}" - }; - await ct.NotifyEvent.AddAsync(n); - log.LogDebug($"Adding NotifyEvent: [{n.ToString()}]"); - await ct.SaveChangesAsync(); - } - } - }// status change event + //not for inactive users + if (!await UserBiz.UserIsActive(sub.UserId)) continue; - //# STATUS AGE - { - //ServerStateStatusAge unchanged for set time (stuck in state), conditional on: Duration (how long stuck), exact status selected IdValue, Tags. Advance notice can NOT be set - //Always clear any old ones for this object as they are all irrelevant the moment the state has changed: - await NotifyEventHelper.ClearPriorEventsForObject(ct, SockType.SubscriptionServer, proposedObj.Id, NotifyEventType.ServerStateStatusAge); - var subs = await ct.NotifySubscription.AsNoTracking().Where(z => z.EventType == NotifyEventType.ServerStateStatusAge && z.IdValue == (long)o.ServerState).ToListAsync(); - foreach (var sub in subs) + //Tag match? (will be true if no sub tags so always safe to call this) + if (NotifyEventHelper.ObjectHasAllSubscriptionTags(ProposedServer.Tags, sub.Tags)) + { + NotifyEvent n = new NotifyEvent() + { + EventType = NotifyEventType.ServerStateStatusChange, + UserId = sub.UserId, + SockType = BizType, + ObjectId = ProposedServer.Id, + NotifySubscriptionId = sub.Id, + Name = $"{ProposedServer.Name} - {ProposedServer.ServerState.ToString()}" + }; + await ct.NotifyEvent.AddAsync(n); + log.LogDebug($"Adding NotifyEvent: [{n.ToString()}]"); + await ct.SaveChangesAsync(); + } + } + }// status change event + + //# STATUS AGE { - //not for inactive users - if (!await UserBiz.UserIsActive(sub.UserId)) continue; - - //WorkOrder Tag match? (Not State, state has no tags, will be true if no sub tags so always safe to call this) - if (NotifyEventHelper.ObjectHasAllSubscriptionTags(o.Tags, sub.Tags)) + //ServerStateStatusAge unchanged for set time (stuck in state), conditional on: Duration (how long stuck), exact status selected IdValue, Tags. Advance notice can NOT be set + //Always clear any old ones for this object as they are all irrelevant the moment the state has changed: + await NotifyEventHelper.ClearPriorEventsForObject(ct, SockType.SubscriptionServer, proposedObj.Id, NotifyEventType.ServerStateStatusAge); + var subs = await ct.NotifySubscription.AsNoTracking().Where(z => z.EventType == NotifyEventType.ServerStateStatusAge && z.IdValue == (long)ProposedServer.ServerState).ToListAsync(); + foreach (var sub in subs) { - NotifyEvent n = new NotifyEvent() - { - EventType = NotifyEventType.ServerStateStatusAge, - UserId = sub.UserId, - SockType = SockType.SubscriptionServer, - ObjectId = o.Id, - NotifySubscriptionId = sub.Id, - Name = $"{o.Name} - {o.ServerState.ToString()}" - }; - await ct.NotifyEvent.AddAsync(n); - log.LogDebug($"Adding NotifyEvent: [{n.ToString()}]"); - await ct.SaveChangesAsync(); - } - } - }// status age event + //not for inactive users + if (!await UserBiz.UserIsActive(sub.UserId)) continue; + //WorkOrder Tag match? (Not State, state has no tags, will be true if no sub tags so always safe to call this) + if (NotifyEventHelper.ObjectHasAllSubscriptionTags(ProposedServer.Tags, sub.Tags)) + { + NotifyEvent n = new NotifyEvent() + { + EventType = NotifyEventType.ServerStateStatusAge, + UserId = sub.UserId, + SockType = SockType.SubscriptionServer, + ObjectId = ProposedServer.Id, + NotifySubscriptionId = sub.Id, + Name = $"{ProposedServer.Name} - {ProposedServer.ServerState.ToString()}" + }; + await ct.NotifyEvent.AddAsync(n); + log.LogDebug($"Adding NotifyEvent: [{n.ToString()}]"); + await ct.SaveChangesAsync(); + } + } + }// status age event + }//is new or ServerState has changed }//object created or modified diff --git a/server/generator/SockBotSubscriptionServerHealthChecks.cs b/server/generator/SockBotSubscriptionServerHealthChecks.cs index 39cc3e0..d11dee4 100644 --- a/server/generator/SockBotSubscriptionServerHealthChecks.cs +++ b/server/generator/SockBotSubscriptionServerHealthChecks.cs @@ -8,7 +8,7 @@ using Sockeye.Models; namespace Sockeye.Biz { - + /// /// Check the health of subscribers servers, basically a ping check /// trigger notification if any fail the test excessively (some slack for intermittent comm. issues) @@ -30,90 +30,31 @@ namespace Sockeye.Biz log.LogDebug("Health check starting"); using (AyContext ct = Sockeye.Util.ServiceProviderProvider.DBContext) { - - var servers = await ct.SubscriptionServer - .AsNoTracking() - .Where(z => z.ServerState== < dtDeleteCutoff && z.JobStatus == jobStatus) - .OrderBy(z => z.Created) - .ToListAsync(); + + var ActiveServerIdList = await ct.SubscriptionServer + .AsNoTracking() + .Where(z => z.ServerState != ServerState.DeActivated && z.ServerState != ServerState.DeActivated) + .OrderBy(z => z.Id) + .Select(z=>z.Id) + .ToListAsync(); + + foreach(long serverId in ActiveServerIdList){ + //get the health and triage accordingly + + + } } lastSweep = DateTime.UtcNow; } - private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus) - { - //Get the deleteable succeeded jobs list - - - log.LogDebug($"SweepAsync processing: cutoff={dtDeleteCutoff.ToString()}, for {jobs.Count.ToString()} jobs of status {jobStatus.ToString()}"); - - foreach (OpsJob j in jobs) - { - try - { - - await JobsBiz.RemoveJobAndLogsAsync(j.GId); - } - catch (Exception ex) - { - log.LogError(ex, "sweepAsync exception calling JobsBiz.RemoveJobAndLogsAsync"); - //for now just throw it but this needs to be removed when logging added and better handling - throw; - } - } - } - /// - /// Kill jobs that have been stuck in "running" state for too long - /// - private static async Task killStuckJobsAsync(AyContext ct, DateTime dtRunningDeadline) - { - //Get the deleteable succeeded jobs list - var jobs = await ct.OpsJob - .AsNoTracking() - .Where(z => z.Created < dtRunningDeadline && z.JobStatus == JobStatus.Running) - .OrderBy(z => z.Created) - .ToListAsync(); - - log.LogDebug($"killStuckJobsAsync processing: cutoff={dtRunningDeadline.ToString()}, for {jobs.Count.ToString()} jobs of status {JobStatus.Running.ToString()}"); - - foreach (OpsJob j in jobs) - { - //OPSMETRIC - await JobsBiz.LogJobAsync(j.GId, "LT:JobFailed LT:TimedOut"); - log.LogError($"Job found job stuck in running status and set to failed: deadline={dtRunningDeadline.ToString()}, jobId={j.GId.ToString()}, jobname={j.Name}, jobtype={j.JobType.ToString()}, jobAType={j.SockType.ToString()}, jobObjectId={j.ObjectId.ToString()}"); - await JobsBiz.UpdateJobStatusAsync(j.GId, JobStatus.Failed); - } - } + - private static async Task SweepInternalJobsLogsAsync(AyContext ct, DateTime dtDeleteCutoff) - { - //Get the deleteable list (this is for reporting, could easily just do it in one go) - var logs = await ct.OpsJobLog - .AsNoTracking() - .Where(z => z.Created < dtDeleteCutoff) - .OrderBy(z => z.Created) - .ToListAsync(); - - log.LogDebug($"SweepInternalJobsLogsAsync processing: cutoff={dtDeleteCutoff.ToString()}, for {logs.Count.ToString()} log entries"); - - foreach (OpsJobLog l in logs) - { - try - { - await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjoblog where gid = {l.GId}"); - } - catch (Exception ex) - { - log.LogError(ex, "SweepInternalJobsLogsAsync exception removed old log entries"); - throw; - } - } - } + ///////////////////////////////////////////////////////////////////// diff --git a/server/util/Mailer.cs b/server/util/Mailer.cs index 1d8de03..cdd113a 100644 --- a/server/util/Mailer.cs +++ b/server/util/Mailer.cs @@ -42,7 +42,7 @@ namespace Sockeye.Util { var message = new MimeMessage(); message.From.Add(new MailboxAddress(smtpSettings.NotifyFromAddress, smtpSettings.NotifyFromAddress)); - message.To.Add(MailboxAddress.Parse(email)); + message.To.Add(MailboxAddress.Parse(email)); message.Subject = subject; if (!string.IsNullOrWhiteSpace(attachPDFPath))