admin/sockeye
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Browse Source
This commit is contained in:
John Cardinal
2023-01-18 03:12:39 +00:00
parent 1ff42db380
commit 126ae54f66
4 changed files with 93 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/biz/NotifyEventType.cs
View File

@@ -34,7 +34,7 @@ Inspiring quotes used to help complete this huge project by myself
ServerStateStatusChange = 36, ServerStateStatusChange = 36,
ServerStateStatusAge=37, ServerStateStatusAge=37,
SubscriptionServerExpiring = 38, SubscriptionServerExpiring = 38,
SubscriptionServerLastUpdateAge = 39, SubscriptionServerLastUpdateAge = 39,//here this means the operating system, not RAVEN which should always be updated on all servers when released
SubscriptionServerRequestReceived = 40, SubscriptionServerRequestReceived = 40,
LicenseTrialRequestReceived = 41, LicenseTrialRequestReceived = 41,
PurchaseNotificationReceived = 42 PurchaseNotificationReceived = 42

144
server/biz/SubscriptionServerBiz.cs
View File

@@ -408,15 +408,15 @@ namespace Sockeye.Biz
await NotifyEventHelper.ProcessStandardObjectEvents(ayaEvent, proposedObj, ct); await NotifyEventHelper.ProcessStandardObjectEvents(ayaEvent, proposedObj, ct);
//SPECIFIC EVENTS FOR THIS OBJECT //SPECIFIC EVENTS FOR THIS OBJECT
SubscriptionServer o = (SubscriptionServer)proposedObj; SubscriptionServer ProposedServer = (SubscriptionServer)proposedObj;
//## DELETED EVENTS //## DELETED EVENTS
//any event added below needs to be removed, so //any event added below needs to be removed, so
//just blanket remove any event for this object of eventtype that would be added below here //just blanket remove any event for this object of eventtype that would be added below here
//do it regardless any time there's an update and then //do it regardless any time there's an update and then
//let this code below handle the refreshing addition that could have changes //let this code below handle the refreshing addition that could have changes
await NotifyEventHelper.ClearPriorEventsForObject(ct, BizType, o.Id, NotifyEventType.SubscriptionServerExpiring); await NotifyEventHelper.ClearPriorEventsForObject(ct, BizType, ProposedServer.Id, NotifyEventType.SubscriptionServerExpiring);
await NotifyEventHelper.ClearPriorEventsForObject(ct, BizType, o.Id, NotifyEventType.SubscriptionServerLastUpdateAge); await NotifyEventHelper.ClearPriorEventsForObject(ct, BizType, ProposedServer.Id, NotifyEventType.SubscriptionServerLastUpdateAge);
//## CREATED / MODIFIED EVENTS //## CREATED / MODIFIED EVENTS
@@ -424,7 +424,7 @@ namespace Sockeye.Biz
{ {
//# NEW TRIAL SERVER REQUEST //# NEW TRIAL SERVER REQUEST
if (ayaEvent == SockEvent.Created && o.Trial == true) if (ayaEvent == SockEvent.Created && ProposedServer.Trial == true)
{ {
{ {
//Conditions: must match specific status id value and also tags below //Conditions: must match specific status id value and also tags below
@@ -436,16 +436,16 @@ namespace Sockeye.Biz
if (!await UserBiz.UserIsActive(sub.UserId)) continue; if (!await UserBiz.UserIsActive(sub.UserId)) continue;
//Tag match? (will be true if no sub tags so always safe to call this) //Tag match? (will be true if no sub tags so always safe to call this)
if (NotifyEventHelper.ObjectHasAllSubscriptionTags(o.Tags, sub.Tags)) if (NotifyEventHelper.ObjectHasAllSubscriptionTags(ProposedServer.Tags, sub.Tags))
{ {
NotifyEvent n = new NotifyEvent() NotifyEvent n = new NotifyEvent()
{ {
EventType = NotifyEventType.SubscriptionServerRequestReceived, EventType = NotifyEventType.SubscriptionServerRequestReceived,
UserId = sub.UserId, UserId = sub.UserId,
SockType = BizType, SockType = BizType,
ObjectId = o.Id, ObjectId = ProposedServer.Id,
NotifySubscriptionId = sub.Id, NotifySubscriptionId = sub.Id,
Name = $"{o.TrialCompany} - trial server requested" Name = $"{ProposedServer.TrialCompany} - trial server requested"
}; };
await ct.NotifyEvent.AddAsync(n); await ct.NotifyEvent.AddAsync(n);
log.LogDebug($"Adding NotifyEvent: [{n.ToString()}]"); log.LogDebug($"Adding NotifyEvent: [{n.ToString()}]");
@@ -466,7 +466,7 @@ namespace Sockeye.Biz
if (!await UserBiz.UserIsActive(sub.UserId)) continue; if (!await UserBiz.UserIsActive(sub.UserId)) continue;
//Tag match? (will be true if no sub tags so always safe to call this) //Tag match? (will be true if no sub tags so always safe to call this)
if (NotifyEventHelper.ObjectHasAllSubscriptionTags(o.Tags, sub.Tags)) if (NotifyEventHelper.ObjectHasAllSubscriptionTags(ProposedServer.Tags, sub.Tags))
{ {
NotifyEvent n = new NotifyEvent() NotifyEvent n = new NotifyEvent()
@@ -474,10 +474,10 @@ namespace Sockeye.Biz
EventType = NotifyEventType.SubscriptionServerExpiring, EventType = NotifyEventType.SubscriptionServerExpiring,
UserId = sub.UserId, UserId = sub.UserId,
SockType = BizType, SockType = BizType,
ObjectId = o.Id, ObjectId = ProposedServer.Id,
NotifySubscriptionId = sub.Id, NotifySubscriptionId = sub.Id,
Name = o.Name, Name = ProposedServer.Name,
EventDate = o.SubscriptionExpire EventDate = ProposedServer.SubscriptionExpire
}; };
await ct.NotifyEvent.AddAsync(n); await ct.NotifyEvent.AddAsync(n);
log.LogDebug($"Adding NotifyEvent: [{n.ToString()}]"); log.LogDebug($"Adding NotifyEvent: [{n.ToString()}]");
@@ -488,9 +488,9 @@ namespace Sockeye.Biz
}//subscription server expiry event }//subscription server expiry event
//# SUBSCRIPTION SERVER LAST UPDATE AGE //# SUBSCRIPTION SERVER LAST UPDATE (O.S.) AGE
{ {
if (o.LastUpdated != null) if (ProposedServer.LastUpdated != null)
{ {
//notify users about warranty expiry (time delayed) //notify users about warranty expiry (time delayed)
var subs = await ct.NotifySubscription.Where(z => z.EventType == NotifyEventType.SubscriptionServerLastUpdateAge).ToListAsync(); var subs = await ct.NotifySubscription.Where(z => z.EventType == NotifyEventType.SubscriptionServerLastUpdateAge).ToListAsync();
@@ -500,17 +500,17 @@ namespace Sockeye.Biz
if (!await UserBiz.UserIsActive(sub.UserId)) continue; if (!await UserBiz.UserIsActive(sub.UserId)) continue;
//Tag match? (will be true if no sub tags so always safe to call this) //Tag match? (will be true if no sub tags so always safe to call this)
if (NotifyEventHelper.ObjectHasAllSubscriptionTags(o.Tags, sub.Tags)) if (NotifyEventHelper.ObjectHasAllSubscriptionTags(ProposedServer.Tags, sub.Tags))
{ {
NotifyEvent n = new NotifyEvent() NotifyEvent n = new NotifyEvent()
{ {
EventType = NotifyEventType.SubscriptionServerLastUpdateAge, EventType = NotifyEventType.SubscriptionServerLastUpdateAge,
UserId = sub.UserId, UserId = sub.UserId,
SockType = BizType, SockType = BizType,
ObjectId = o.Id, ObjectId = ProposedServer.Id,
NotifySubscriptionId = sub.Id, NotifySubscriptionId = sub.Id,
Name = o.Name, Name = ProposedServer.Name,
EventDate = (DateTime)o.LastUpdated EventDate = (DateTime)ProposedServer.LastUpdated
}; };
await ct.NotifyEvent.AddAsync(n); await ct.NotifyEvent.AddAsync(n);
log.LogDebug($"Adding NotifyEvent: [{n.ToString()}]"); log.LogDebug($"Adding NotifyEvent: [{n.ToString()}]");
@@ -522,66 +522,70 @@ namespace Sockeye.Biz
}//subscription server last updated event }//subscription server last updated event
//##################################
//# STATUS CHANGE (create new status) //ServerState changes events
//##################################
if (isNew || ((SubscriptionServer)currentObj).ServerState != ProposedServer.ServerState)//must have changed from a prior state
{ {
//Conditions: must match specific status id value and also tags below //# STATUS CHANGE (changed to this new status)
//delivery is immediate so no need to remove old ones of this kind
var subs = await ct.NotifySubscription.AsNoTracking().Where(z => z.EventType == NotifyEventType.ServerStateStatusChange && z.IdValue == (long)o.ServerState).ToListAsync();
foreach (var sub in subs)
{ {
//not for inactive users //Conditions: must match specific status id value and also tags below
if (!await UserBiz.UserIsActive(sub.UserId)) continue; //delivery is immediate so no need to remove old ones of this kind
var subs = await ct.NotifySubscription.AsNoTracking().Where(z => z.EventType == NotifyEventType.ServerStateStatusChange && z.IdValue == (long)ProposedServer.ServerState).ToListAsync();
//Tag match? (will be true if no sub tags so always safe to call this) foreach (var sub in subs)
if (NotifyEventHelper.ObjectHasAllSubscriptionTags(o.Tags, sub.Tags))
{ {
NotifyEvent n = new NotifyEvent() //not for inactive users
{ if (!await UserBiz.UserIsActive(sub.UserId)) continue;
EventType = NotifyEventType.ServerStateStatusChange,
UserId = sub.UserId,
SockType = BizType,
ObjectId = o.Id,
NotifySubscriptionId = sub.Id,
Name = $"{o.Name} - {o.ServerState.ToString()}"
};
await ct.NotifyEvent.AddAsync(n);
log.LogDebug($"Adding NotifyEvent: [{n.ToString()}]");
await ct.SaveChangesAsync();
}
}
}// status change event
//# STATUS AGE //Tag match? (will be true if no sub tags so always safe to call this)
{ if (NotifyEventHelper.ObjectHasAllSubscriptionTags(ProposedServer.Tags, sub.Tags))
//ServerStateStatusAge unchanged for set time (stuck in state), conditional on: Duration (how long stuck), exact status selected IdValue, Tags. Advance notice can NOT be set {
//Always clear any old ones for this object as they are all irrelevant the moment the state has changed: NotifyEvent n = new NotifyEvent()
await NotifyEventHelper.ClearPriorEventsForObject(ct, SockType.SubscriptionServer, proposedObj.Id, NotifyEventType.ServerStateStatusAge); {
var subs = await ct.NotifySubscription.AsNoTracking().Where(z => z.EventType == NotifyEventType.ServerStateStatusAge && z.IdValue == (long)o.ServerState).ToListAsync(); EventType = NotifyEventType.ServerStateStatusChange,
foreach (var sub in subs) UserId = sub.UserId,
SockType = BizType,
ObjectId = ProposedServer.Id,
NotifySubscriptionId = sub.Id,
Name = $"{ProposedServer.Name} - {ProposedServer.ServerState.ToString()}"
};
await ct.NotifyEvent.AddAsync(n);
log.LogDebug($"Adding NotifyEvent: [{n.ToString()}]");
await ct.SaveChangesAsync();
}
}
}// status change event
//# STATUS AGE
{ {
//not for inactive users //ServerStateStatusAge unchanged for set time (stuck in state), conditional on: Duration (how long stuck), exact status selected IdValue, Tags. Advance notice can NOT be set
if (!await UserBiz.UserIsActive(sub.UserId)) continue; //Always clear any old ones for this object as they are all irrelevant the moment the state has changed:
await NotifyEventHelper.ClearPriorEventsForObject(ct, SockType.SubscriptionServer, proposedObj.Id, NotifyEventType.ServerStateStatusAge);
//WorkOrder Tag match? (Not State, state has no tags, will be true if no sub tags so always safe to call this) var subs = await ct.NotifySubscription.AsNoTracking().Where(z => z.EventType == NotifyEventType.ServerStateStatusAge && z.IdValue == (long)ProposedServer.ServerState).ToListAsync();
if (NotifyEventHelper.ObjectHasAllSubscriptionTags(o.Tags, sub.Tags)) foreach (var sub in subs)
{ {
NotifyEvent n = new NotifyEvent() //not for inactive users
{ if (!await UserBiz.UserIsActive(sub.UserId)) continue;
EventType = NotifyEventType.ServerStateStatusAge,
UserId = sub.UserId,
SockType = SockType.SubscriptionServer,
ObjectId = o.Id,
NotifySubscriptionId = sub.Id,
Name = $"{o.Name} - {o.ServerState.ToString()}"
};
await ct.NotifyEvent.AddAsync(n);
log.LogDebug($"Adding NotifyEvent: [{n.ToString()}]");
await ct.SaveChangesAsync();
}
}
}// status age event
//WorkOrder Tag match? (Not State, state has no tags, will be true if no sub tags so always safe to call this)
if (NotifyEventHelper.ObjectHasAllSubscriptionTags(ProposedServer.Tags, sub.Tags))
{
NotifyEvent n = new NotifyEvent()
{
EventType = NotifyEventType.ServerStateStatusAge,
UserId = sub.UserId,
SockType = SockType.SubscriptionServer,
ObjectId = ProposedServer.Id,
NotifySubscriptionId = sub.Id,
Name = $"{ProposedServer.Name} - {ProposedServer.ServerState.ToString()}"
};
await ct.NotifyEvent.AddAsync(n);
log.LogDebug($"Adding NotifyEvent: [{n.ToString()}]");
await ct.SaveChangesAsync();
}
}
}// status age event
}//is new or ServerState has changed
}//object created or modified }//object created or modified

83
server/generator/SockBotSubscriptionServerHealthChecks.cs
View File

@@ -31,89 +31,30 @@ namespace Sockeye.Biz
using (AyContext ct = Sockeye.Util.ServiceProviderProvider.DBContext) using (AyContext ct = Sockeye.Util.ServiceProviderProvider.DBContext)
{ {
var servers = await ct.SubscriptionServer var ActiveServerIdList = await ct.SubscriptionServer
.AsNoTracking() .AsNoTracking()
.Where(z => z.ServerState== < dtDeleteCutoff && z.JobStatus == jobStatus) .Where(z => z.ServerState != ServerState.DeActivated && z.ServerState != ServerState.DeActivated)
.OrderBy(z => z.Created) .OrderBy(z => z.Id)
.ToListAsync(); .Select(z=>z.Id)
.ToListAsync();
foreach(long serverId in ActiveServerIdList){
//get the health and triage accordingly
}
} }
lastSweep = DateTime.UtcNow; lastSweep = DateTime.UtcNow;
} }
private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus)
{
//Get the deleteable succeeded jobs list
log.LogDebug($"SweepAsync processing: cutoff={dtDeleteCutoff.ToString()}, for {jobs.Count.ToString()} jobs of status {jobStatus.ToString()}");
foreach (OpsJob j in jobs)
{
try
{
await JobsBiz.RemoveJobAndLogsAsync(j.GId);
}
catch (Exception ex)
{
log.LogError(ex, "sweepAsync exception calling JobsBiz.RemoveJobAndLogsAsync");
//for now just throw it but this needs to be removed when logging added and better handling
throw;
}
}
}
/// <summary>
/// Kill jobs that have been stuck in "running" state for too long
/// </summary>
private static async Task killStuckJobsAsync(AyContext ct, DateTime dtRunningDeadline)
{
//Get the deleteable succeeded jobs list
var jobs = await ct.OpsJob
.AsNoTracking()
.Where(z => z.Created < dtRunningDeadline && z.JobStatus == JobStatus.Running)
.OrderBy(z => z.Created)
.ToListAsync();
log.LogDebug($"killStuckJobsAsync processing: cutoff={dtRunningDeadline.ToString()}, for {jobs.Count.ToString()} jobs of status {JobStatus.Running.ToString()}");
foreach (OpsJob j in jobs)
{
//OPSMETRIC
await JobsBiz.LogJobAsync(j.GId, "LT:JobFailed LT:TimedOut");
log.LogError($"Job found job stuck in running status and set to failed: deadline={dtRunningDeadline.ToString()}, jobId={j.GId.ToString()}, jobname={j.Name}, jobtype={j.JobType.ToString()}, jobAType={j.SockType.ToString()}, jobObjectId={j.ObjectId.ToString()}");
await JobsBiz.UpdateJobStatusAsync(j.GId, JobStatus.Failed);
}
}
private static async Task SweepInternalJobsLogsAsync(AyContext ct, DateTime dtDeleteCutoff)
{
//Get the deleteable list (this is for reporting, could easily just do it in one go)
var logs = await ct.OpsJobLog
.AsNoTracking()
.Where(z => z.Created < dtDeleteCutoff)
.OrderBy(z => z.Created)
.ToListAsync();
log.LogDebug($"SweepInternalJobsLogsAsync processing: cutoff={dtDeleteCutoff.ToString()}, for {logs.Count.ToString()} log entries");
foreach (OpsJobLog l in logs)
{
try
{
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjoblog where gid = {l.GId}");
}
catch (Exception ex)
{
log.LogError(ex, "SweepInternalJobsLogsAsync exception removed old log entries");
throw;
}
}
}
///////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////