admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

much fuckery

Browse Source
This commit is contained in:
John Cardinal
2020-05-25 20:34:11 +00:00
parent d977728a22
commit e90dbe0f71
9 changed files with 218 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
server/AyaNova/generator/CoreJobSweeper.cs
View File

@@ -4,7 +4,7 @@ using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using AyaNova.Models;
using System.Threading;
namespace AyaNova.Biz
{
@@ -28,44 +28,49 @@ namespace AyaNova.Biz
////////////////////////////////////////////////////////////////////////////////////////////////
// DoSweep
//
public static async Task DoSweepAsync(AyContext ct)
public static async Task DoSweepAsync(CancellationToken ctoken)
{
ctoken.ThrowIfCancellationRequested();
//This will get triggered roughly every minute, but we don't want to sweep that frequently
if (DateTime.UtcNow - lastSweep < SWEEP_EVERY_INTERVAL)
return;
log.LogTrace("Sweep starting");
using (AyContext ct = AyaNova.Util.ServiceProviderProvider.DBContext)
{
//SWEEP SUCCESSFUL JOBS
//calculate cutoff to delete
DateTime dtDeleteCutoff = DateTime.UtcNow - SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Completed);
//SWEEP SUCCESSFUL JOBS
//calculate cutoff to delete
DateTime dtDeleteCutoff = DateTime.UtcNow - SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Completed, ctoken);
//SWEEP FAILED JOBS
//calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Failed);
//KILL STUCK JOBS
//calculate cutoff to delete
DateTime dtRunningDeadline = DateTime.UtcNow - RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN;
await killStuckJobsAsync(ct, dtRunningDeadline);
//SWEEP INTERNAL JOB LOG
//calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - INTERNAL_JOBS_LOGS_DELETE_AFTER_THIS_TIMESPAN;
await SweepInternalJobsLogsAsync(ct, dtDeleteCutoff);
ctoken.ThrowIfCancellationRequested();
//SWEEP FAILED JOBS
//calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Failed, ctoken);
ctoken.ThrowIfCancellationRequested();
//KILL STUCK JOBS
//calculate cutoff to delete
DateTime dtRunningDeadline = DateTime.UtcNow - RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN;
await killStuckJobsAsync(ct, dtRunningDeadline);
ctoken.ThrowIfCancellationRequested();
//SWEEP INTERNAL JOB LOG
//calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - INTERNAL_JOBS_LOGS_DELETE_AFTER_THIS_TIMESPAN;
await SweepInternalJobsLogsAsync(ct, dtDeleteCutoff);
ctoken.ThrowIfCancellationRequested();
}
lastSweep = DateTime.UtcNow;
}
private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus)
private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus, CancellationToken ctoken)//AyContext ct,
{
// AyContext ct = ServiceProviderProvider.DBContext;
//Get the deleteable succeeded jobs list
var jobs = await ct.OpsJob
.AsNoTracking()
@@ -79,11 +84,12 @@ namespace AyaNova.Biz
{
try
{
await JobsBiz.DeleteJobAndLogAsync(j.GId, ct);
ctoken.ThrowIfCancellationRequested();
await JobsBiz.RemoveJobAndLogsAsync(j.GId);
}
catch (Exception ex)
{
log.LogError(ex, "sweepAsync exception calling JobsBiz.DeleteJobAndLogAsync");
log.LogError(ex, "sweepAsync exception calling JobsBiz.RemoveJobAndLogsAsync");
//for now just throw it but this needs to be removed when logging added and better handling
throw (ex);
}
@@ -93,12 +99,13 @@ namespace AyaNova.Biz
/// <summary>
/// Kill jobs that have been stuck in "running" state for too long
/// </summary>
/// <param name="ct"></param>
/// </summary>
/// <param name="dtRunningDeadline"></param>
/// <returns></returns>
private static async Task killStuckJobsAsync(AyContext ct, DateTime dtRunningDeadline)
{
// AyContext ct = ServiceProviderProvider.DBContext;
//Get the deleteable succeeded jobs list
var jobs = await ct.OpsJob
.AsNoTracking()
@@ -120,7 +127,7 @@ namespace AyaNova.Biz
private static async Task SweepInternalJobsLogsAsync(AyContext ct, DateTime dtDeleteCutoff)
{
// AyContext ct = ServiceProviderProvider.DBContext;
//Get the deleteable list (this is for reporting, could easily just do it in one go)
var logs = await ct.OpsJobLog
.AsNoTracking()