This commit is contained in:
2020-05-19 18:27:52 +00:00
parent d20bd3a50e
commit 24c53a3d26
9 changed files with 75 additions and 72 deletions

View File

@@ -4,6 +4,7 @@ using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using AyaNova.Models;
using AyaNova.Util;
namespace AyaNova.Biz
@@ -17,44 +18,43 @@ namespace AyaNova.Biz
internal static class CoreJobBackup
{
private static ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("CoreJobBackup");
private static DateTime lastSweep = DateTime.MinValue;
private static TimeSpan SWEEP_EVERY_INTERVAL = new TimeSpan(0, 30, 0);
private static TimeSpan SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN = new TimeSpan(24, 0, 0);//24 hours
private static TimeSpan FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN = new TimeSpan(14, 0, 0, 0);//14 days (gives people time to notice and look into it)
private static TimeSpan RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN = new TimeSpan(24, 0, 0);//24 hours (time running jobs are allowed to sit in "running" state before considered failed)
private static bool BackupIsRunning = false;
////////////////////////////////////////////////////////////////////////////////////////////////
//
//
public static async Task DoWorkAsync(AyContext ct)
{
if (BackupIsRunning) return;
//This will get triggered roughly every minute, but we don't want to sweep that frequently
if (DateTime.UtcNow - lastSweep < SWEEP_EVERY_INTERVAL)
return;
//get NOW in utc
DateTime utcNow = DateTime.UtcNow;
//what time should we backup today?
DateTime todayBackupTime = new DateTime(utcNow.Year, utcNow.Month, utcNow.Day, ServerGlobalOpsSettings.BackupTime.Hour, ServerGlobalOpsSettings.BackupTime.Minute, 0, DateTimeKind.Utc);//first start with NOW
//Are we there yet?
if (utcNow < todayBackupTime) return;//nope
//Yes, we've passed into the backup window time, but that's also true if we just ran the backup as well so
//need to check for that as well...
//Has last backup run more than 24 hours ago?
if (ServerGlobalOpsSettings.LastBackup > utcNow.AddHours(-24))
return;//nope, so we have already run today's backup
//Ok, we're into backup time and it's been more than 24 hours since it last ran so let's do this...
BackupIsRunning = true;
log.LogTrace("Backup starting");
//lock the server
//serverState.SetSystemLock(msg);
//SWEEP SUCCESSFUL JOBS
//calculate cutoff to delete
DateTime dtDeleteCutoff = DateTime.UtcNow - SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Completed);
//SWEEP FAILED JOBS
//calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Failed);
//*************
//BACKUP TORA TORA TORA!
//***************
//KILL STUCK JOBS
//calculate cutoff to delete
DateTime dtRunningDeadline = DateTime.UtcNow - RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN;
await killStuckJobsAsync(ct, dtRunningDeadline);
lastSweep = DateTime.UtcNow;
//Update last backup
BackupIsRunning = false;
log.LogTrace("Backup completed");
}