admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Browse Source
This commit is contained in:
John Cardinal
2020-06-15 14:48:45 +00:00
parent f80d1bbca0
commit e7b5e1a388
5 changed files with 77 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

177
server/AyaNova/generator/CoreJobLicense.cs
View File

@@ -4,6 +4,8 @@ using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using AyaNova.Models;
using AyaNova.Core;
using AyaNova.Util;
namespace AyaNova.Biz
{
@@ -16,141 +18,74 @@ namespace AyaNova.Biz
internal static class CoreJobLicense
{
private static ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("CoreJobLicense");
private static DateTime lastCheck = DateTime.MinValue;
//define timespans for various existing license scenarios
//120 days after a new license purchase it should check for revoke from chargebacks
//Trial should check regularly for purchase
//revoked should do what?
//rental licenses need to check more often than perpetual
//slow and fast scenarios maybe? Just two timespans, fast for likely purchaser and slow for ongoing rental maint etc
private static TimeSpan SWEEP_EVERY_INTERVAL = new TimeSpan(0, 30, 0);
private static TimeSpan SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN = new TimeSpan(14, 0, 0, 0);//14 days
private static TimeSpan FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN = new TimeSpan(14, 0, 0, 0);//14 days (gives people time to notice and look into it)
private static TimeSpan INTERNAL_JOBS_LOGS_DELETE_AFTER_THIS_TIMESPAN = new TimeSpan(14, 0, 0, 0);//14 days
private static TimeSpan RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN = new TimeSpan(24, 0, 0);//24 hours (time running jobs are allowed to sit in "running" state before considered failed)
private static DateTime _lastCheck = DateTime.MinValue;
private static TimeSpan FAST_TRACK = new TimeSpan(0, 0, 30, 0);
private static TimeSpan SLOW_TRACK = new TimeSpan(24, 0, 0);
////////////////////////////////////////////////////////////////////////////////////////////////
// DoSweep
// DoWork
//
public static async Task DoSweepAsync()
{
//This will get triggered roughly every minute, but we don't want to sweep that frequently
if (DateTime.UtcNow - lastCheck < SWEEP_EVERY_INTERVAL)
public static async Task DoWorkAsync()
{
log.LogTrace("Job starting");
var tsSinceLastCheck = DateTime.UtcNow - _lastCheck;
//which track are we on?
/*
NONE = 0,//fast track
ActiveTrial = 1,//slow track
ExpiredTrial = 2,//fast track
ActivePurchased = 3,//slow track
ExpiredPurchased = 4,//fast track
Revoked = 5//slow track
*/
TimeSpan tsCheckFrequency;
switch (AyaNova.Core.License.ActiveKey.Status)
{
case AyaNova.Core.License.AyaNovaLicenseKey.LicenseStatus.NONE:
case AyaNova.Core.License.AyaNovaLicenseKey.LicenseStatus.ExpiredTrial:
case AyaNova.Core.License.AyaNovaLicenseKey.LicenseStatus.ExpiredPurchased:
tsCheckFrequency = FAST_TRACK;
break;
default:
tsCheckFrequency = SLOW_TRACK;
break;
}
log.LogTrace($"Check frequency{tsCheckFrequency}");
if (tsSinceLastCheck < tsCheckFrequency)
{
log.LogTrace($"Not yet");
return;
}
log.LogTrace("Sweep starting");
using (AyContext ct = AyaNova.Util.ServiceProviderProvider.DBContext)
{
//SWEEP SUCCESSFUL JOBS
//calculate cutoff to delete
DateTime dtDeleteCutoff = DateTime.UtcNow - SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Completed);
//SWEEP FAILED JOBS
//calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Failed);
//KILL STUCK JOBS
//calculate cutoff to delete
DateTime dtRunningDeadline = DateTime.UtcNow - RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN;
await killStuckJobsAsync(ct, dtRunningDeadline);
//SWEEP INTERNAL JOB LOG
//calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - INTERNAL_JOBS_LOGS_DELETE_AFTER_THIS_TIMESPAN;
await SweepInternalJobsLogsAsync(ct, dtDeleteCutoff);
}
lastCheck = DateTime.UtcNow;
}
private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus)
{
//Get the deleteable succeeded jobs list
var jobs = await ct.OpsJob
.AsNoTracking()
.Where(z => z.Created < dtDeleteCutoff && z.JobStatus == jobStatus)
.OrderBy(z => z.Created)
.ToListAsync();
log.LogTrace($"SweepAsync processing: cutoff={dtDeleteCutoff.ToString()}, for {jobs.Count.ToString()} jobs of status {jobStatus.ToString()}");
foreach (OpsJob j in jobs)
{
try
try//allow exception to bubble up but using try block to ensure the lastcheck is set properly regardless so this doesn't spin out of control
{
await JobsBiz.RemoveJobAndLogsAsync(j.GId);
var ServerState = (AyaNova.Api.ControllerHelpers.ApiServerState)ServiceProviderProvider.Provider.GetService(typeof(AyaNova.Api.ControllerHelpers.ApiServerState));
var ret = await AyaNova.Core.License.FetchKeyAsync(ServerState, ct, log, true);
//most often the result will be "notfound" but in future might be other results
if (ret == "ok")
{
//Eventlog
await EventLogProcessor.LogEventToDatabaseAsync(new Event(1, 0, AyaType.License, AyaEvent.LicenseFetch, "FromCoreJob"), ct);
}
if(ret!="notfound")
{
//I'm thinking do not log internally failed except as trace event as this would fill up log file
//instead if they have a license issue they can do manual fetch and then that will log so they can see error
//(also it will return an error to the client)
log.LogTrace($"FetchLicense - failed: {ret}");
}
}
catch (Exception ex)
finally
{
log.LogError(ex, "sweepAsync exception calling JobsBiz.RemoveJobAndLogsAsync");
//for now just throw it but this needs to be removed when logging added and better handling
throw (ex);
_lastCheck = DateTime.UtcNow;
}
}
}
/// <summary>
/// Kill jobs that have been stuck in "running" state for too long
/// </summary>
private static async Task killStuckJobsAsync(AyContext ct, DateTime dtRunningDeadline)
{
//Get the deleteable succeeded jobs list
var jobs = await ct.OpsJob
.AsNoTracking()
.Where(z => z.Created < dtRunningDeadline && z.JobStatus == JobStatus.Running)
.OrderBy(z => z.Created)
.ToListAsync();
log.LogTrace($"killStuckJobsAsync processing: cutoff={dtRunningDeadline.ToString()}, for {jobs.Count.ToString()} jobs of status {JobStatus.Running.ToString()}");
foreach (OpsJob j in jobs)
{
//OPSMETRIC
await JobsBiz.LogJobAsync(j.GId, "Job took too long to run - setting to failed");
log.LogError($"Job found job stuck in running status and set to failed: deadline={dtRunningDeadline.ToString()}, jobId={j.GId.ToString()}, jobname={j.Name}, jobtype={j.JobType.ToString()}, jobObjectType={j.ObjectType.ToString()}, jobObjectId={j.ObjectId.ToString()}");
await JobsBiz.UpdateJobStatusAsync(j.GId, JobStatus.Failed);
}
}
private static async Task SweepInternalJobsLogsAsync(AyContext ct, DateTime dtDeleteCutoff)
{
//Get the deleteable list (this is for reporting, could easily just do it in one go)
var logs = await ct.OpsJobLog
.AsNoTracking()
.Where(z => z.Created < dtDeleteCutoff)
.OrderBy(z => z.Created)
.ToListAsync();
log.LogTrace($"SweepInternalJobsLogsAsync processing: cutoff={dtDeleteCutoff.ToString()}, for {logs.Count.ToString()} log entries");
foreach (OpsJobLog l in logs)
{
try
{
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjoblog where gid = {l.GId}");
}
catch (Exception ex)
{
log.LogError(ex, "SweepInternalJobsLogsAsync exception removed old log entries");
throw (ex);
}
}
}
/////////////////////////////////////////////////////////////////////
}//eoc
}//eons