using System; using System.Linq; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using AyaNova.Models; using AyaNova.Util; namespace AyaNova.Biz { /// /// Backup /// /// internal static class CoreJobBackup { private static ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("CoreJobBackup"); private static bool BackupIsRunning = false; //////////////////////////////////////////////////////////////////////////////////////////////// // // public static async Task DoWorkAsync(AyContext ct) { if (BackupIsRunning) return; //get NOW in utc DateTime utcNow = DateTime.UtcNow; //what time should we backup today? DateTime todayBackupTime = new DateTime(utcNow.Year, utcNow.Month, utcNow.Day, ServerGlobalOpsSettings.BackupTime.Hour, ServerGlobalOpsSettings.BackupTime.Minute, 0, DateTimeKind.Utc);//first start with NOW //Are we there yet? if (utcNow < todayBackupTime) return;//nope //Yes, we've passed into the backup window time, but that's also true if we just ran the backup as well so //need to check for that as well... //Has last backup run more than 24 hours ago? if (ServerGlobalOpsSettings.LastBackup > utcNow.AddHours(-24)) return;//nope, so we have already run today's backup //Ok, we're into backup time and it's been more than 24 hours since it last ran so let's do this... AyaNova.Api.ControllerHelpers.ApiServerState apiServerState = null; try { BackupIsRunning = true; //LOCK DOWN SERVER apiServerState = (AyaNova.Api.ControllerHelpers.ApiServerState)ServiceProviderProvider.Provider.GetService(typeof(AyaNova.Api.ControllerHelpers.ApiServerState)); apiServerState.SetClosed("BACKUP JOB RUNNING"); log.LogTrace("Backup starting"); //************* //DO DATA BACKUP log.LogInformation("SIMULATED BACKUP RUNNING NOW - TORA TORA TORA!"); log.LogInformation($"dbdump path: {ServerBootConfig.AYANOVA_BACKUP_PG_DUMP_PATH}"); //DO FILE BACKUP IF ATTACHMENTS BACKED UP //OPNE SERVER //PRUNE BACKUP SETS NOT KEPT //v.next - COPY TO ONLINE STORAGE //*************** //Update last backup var biz = GlobalOpsSettingsBiz.GetBiz(ct); var OpSet = await biz.GetAsync(false); OpSet.LastBackup = utcNow; await biz.ReplaceAsync(OpSet); log.LogTrace("Backup completed"); } catch (Exception ex) { log.LogError(ex, "BACKUP FAILED!"); throw ex; } finally { apiServerState.ResumePriorState(); BackupIsRunning = false; } } private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus) { //Get the deleteable succeeded jobs list var jobs = await ct.OpsJob .AsNoTracking() .Where(z => z.Created < dtDeleteCutoff && z.JobStatus == jobStatus) .OrderBy(z => z.Created) .ToListAsync(); log.LogTrace($"SweepAsync processing: cutoff={dtDeleteCutoff.ToString()}, for {jobs.Count.ToString()} jobs of status {jobStatus.ToString()}"); foreach (OpsJob j in jobs) { try { await JobsBiz.DeleteJobAndLogAsync(j.GId, ct); } catch (Exception ex) { log.LogError(ex, "sweepAsync exception calling JobsBiz.DeleteJobAndLogAsync"); //for now just throw it but this needs to be removed when logging added and better handling throw (ex); } } } /// /// Kill jobs that have been stuck in "running" state for too long /// /// /// /// private static async Task killStuckJobsAsync(AyContext ct, DateTime dtRunningDeadline) { //Get the deleteable succeeded jobs list var jobs = await ct.OpsJob .AsNoTracking() .Where(z => z.Created < dtRunningDeadline && z.JobStatus == JobStatus.Running) .OrderBy(z => z.Created) .ToListAsync(); log.LogTrace($"killStuckJobsAsync processing: cutoff={dtRunningDeadline.ToString()}, for {jobs.Count.ToString()} jobs of status {JobStatus.Running.ToString()}"); foreach (OpsJob j in jobs) { //OPSMETRIC await JobsBiz.LogJobAsync(j.GId, "Job took too long to run - setting to failed", ct); log.LogError($"Job found job stuck in running status and set to failed: deadline={dtRunningDeadline.ToString()}, jobId={j.GId.ToString()}, jobname={j.Name}, jobtype={j.JobType.ToString()}, jobObjectType={j.ObjectType.ToString()}, jobObjectId={j.ObjectId.ToString()}"); await JobsBiz.UpdateJobStatusAsync(j.GId, JobStatus.Failed, ct); } } ///////////////////////////////////////////////////////////////////// }//eoc }//eons