From a10a3e3069a7c63d612e2bf2b6687a76d4313ef2 Mon Sep 17 00:00:00 2001 From: John Cardinal Date: Mon, 25 May 2020 22:22:34 +0000 Subject: [PATCH] --- server/AyaNova/biz/JobsBiz.cs | 9 ++++- server/AyaNova/generator/CoreJobBackup.cs | 20 ++++++----- server/AyaNova/generator/Generate.cs | 2 -- server/AyaNova/util/RunProgram.cs | 43 ++++++++++++++--------- 4 files changed, 45 insertions(+), 29 deletions(-) diff --git a/server/AyaNova/biz/JobsBiz.cs b/server/AyaNova/biz/JobsBiz.cs index 24249ecc..f16b6c6d 100644 --- a/server/AyaNova/biz/JobsBiz.cs +++ b/server/AyaNova/biz/JobsBiz.cs @@ -338,13 +338,20 @@ namespace AyaNova.Biz } //STOCK JOBS - + //Capture metrics + CoreJobMetricsSnapshot.DoJob(); //Notifications } + catch (OperationCanceledException e) + { + log.LogError(e, "JobsBiz::ProcessJobsAsync jobs cancelled likely due to timeout"); + System.Diagnostics.Debug.WriteLine($"JobsBiz::ProcessJobsAsync {nameof(OperationCanceledException)} thrown with message: {e.Message}"); + throw; + } catch (Exception ex) { log.LogError(ex, "JobsBiz::ProcessJobsAsync unexpected error during processing"); diff --git a/server/AyaNova/generator/CoreJobBackup.cs b/server/AyaNova/generator/CoreJobBackup.cs index 9912ad03..a0700d27 100644 --- a/server/AyaNova/generator/CoreJobBackup.cs +++ b/server/AyaNova/generator/CoreJobBackup.cs @@ -19,6 +19,7 @@ namespace AyaNova.Biz { private static ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("CoreJobBackup"); private static bool BackupIsRunning = false; + private const int MAXIMUM_MS_ALLOWED_FOR_PROCESSING = 15 * 60 * 1000;//wild assed guess 15 minutes maximum to run backup command, ever //////////////////////////////////////////////////////////////////////////////////////////////// // // @@ -26,18 +27,18 @@ namespace AyaNova.Biz { //## TEST TEMPORARY //This should trigger the kill switch for generator - System.Diagnostics.Debug.WriteLine($"CoreJobBackup test wait 5 minutes starting now..."); - await Task.Delay(new TimeSpan(0,5,0)); + System.Diagnostics.Debug.WriteLine($"CoreJobBackup test wait 2 minutes starting now..."); + await Task.Delay(new TimeSpan(0, 2, 0)); if (BackupIsRunning) return; if (!OnDemand) { - log.LogTrace("Checking if backup should run"); + log.LogTrace("Checking if backup should run"); if (DateTime.UtcNow < ServerGlobalOpsSettingsCache.NextBackup) { - log.LogTrace("Not past backup time yet"); + log.LogTrace("Not past backup time yet"); return; - } + } } AyaNova.Api.ControllerHelpers.ApiServerState apiServerState = null; try @@ -48,7 +49,7 @@ namespace AyaNova.Biz apiServerState.SetClosed("BACKUP RUNNING"); await JobsBiz.LogJobAsync(Guid.Empty, $"Starting backup job {(OnDemand ? "manual / on demand" : "scheduled") } "); log.LogDebug("Backup starting"); - var DemandFileNamePrepend=OnDemand?"manual-":string.Empty; + var DemandFileNamePrepend = OnDemand ? "manual-" : string.Empty; //************* //DO DATA BACKUP //build command @@ -68,16 +69,16 @@ namespace AyaNova.Biz var Arguments = $"{DBNameParameter} -Fc > {DataBackupFile}"; - var Result = RunProgram.Run(BackupUtilityCommand, Arguments, log); + var Result = RunProgram.Run(BackupUtilityCommand, Arguments, log, MAXIMUM_MS_ALLOWED_FOR_PROCESSING); if (string.IsNullOrWhiteSpace(Result)) { - log.LogDebug("BACKUP SUCCESSFUL (NO ERROR)"); } else { await JobsBiz.LogJobAsync(Guid.Empty, $"Error during data backup \"{Result}\""); log.LogError($"BACKUP ERROR: {Result}"); + //TODO:OPSNOTIFY } //DO FILE BACKUP IF ATTACHMENTS BACKED UP @@ -100,12 +101,13 @@ namespace AyaNova.Biz await JobsBiz.LogJobAsync(Guid.Empty, "Backup failed with errors:"); await JobsBiz.LogJobAsync(Guid.Empty, ExceptionUtil.ExtractAllExceptionMessages(ex)); log.LogError(ex, "Backup failed"); + //TODO:OPSNOTIFY throw ex; } finally { //bump the backup date if automatic backup - if(!OnDemand) + if (!OnDemand) ServerGlobalOpsSettingsCache.SetNextBackup(); apiServerState.ResumePriorState(); BackupIsRunning = false; diff --git a/server/AyaNova/generator/Generate.cs b/server/AyaNova/generator/Generate.cs index aa3d41ba..44b2c23c 100644 --- a/server/AyaNova/generator/Generate.cs +++ b/server/AyaNova/generator/Generate.cs @@ -2,8 +2,6 @@ using System.Threading; using System; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.DependencyInjection; -using AyaNova.Models; using AyaNova.Api.ControllerHelpers; using AyaNova.Biz; using AyaNova.Util; diff --git a/server/AyaNova/util/RunProgram.cs b/server/AyaNova/util/RunProgram.cs index 38b0d127..44a38a4a 100644 --- a/server/AyaNova/util/RunProgram.cs +++ b/server/AyaNova/util/RunProgram.cs @@ -10,22 +10,22 @@ namespace AyaNova.Util { - public static string Run(string cmd, string arguments, ILogger log = null) + public static string Run(string cmd, string arguments, ILogger log = null, int waitForExitTimeOut = int.MaxValue) { try { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - return RunWindows(cmd, arguments); + return RunWindows(cmd, arguments, waitForExitTimeOut); } else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { - return RunOSX(cmd, arguments); + return RunOSX(cmd, arguments, waitForExitTimeOut); } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { - return RunLinuxBash(cmd, arguments); + return RunLinuxBash(cmd, arguments, waitForExitTimeOut); } } catch (Exception ex) @@ -46,35 +46,38 @@ namespace AyaNova.Util - private static string RunWindows(string cmd, string arguments) + private static string RunWindows(string cmd, string arguments, int waitForExitTimeOut = int.MaxValue) { - //RunProgram.Run("cmd.exe",FullRunCommand, log); - // var FullRunCommand=$"/C {BackupUtilityCommand} {Arguments}"; - -//for Windows need to pass to cmd.exe because often have command line piping etc - var args=$"/C {cmd} {arguments}"; + //RunProgram.Run("cmd.exe",FullRunCommand, log); + // var FullRunCommand=$"/C {BackupUtilityCommand} {Arguments}"; + + //for Windows need to pass to cmd.exe because often have command line piping etc + var args = $"/C {cmd} {arguments}"; using (var process = new Process()) { process.StartInfo = new ProcessStartInfo { FileName = "cmd.exe", - Arguments = args, + Arguments = args, RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true, }; - + process.Start(); string result = $"{process.StandardOutput.ReadToEnd()}{process.StandardError.ReadToEnd()} "; - process.WaitForExit(); + if (!process.WaitForExit(waitForExitTimeOut)) + { + result += $"\nERROR: TIMED OUT {waitForExitTimeOut}ms BEFORE COMPLETION"; + } return result; } } - private static string RunLinuxBash(string cmd, string arguments) + private static string RunLinuxBash(string cmd, string arguments, int waitForExitTimeOut = int.MaxValue) { var escapedArgs = $"{cmd} {arguments}".Replace("\"", "\\\""); @@ -91,13 +94,16 @@ namespace AyaNova.Util process.Start(); string result = process.StandardOutput.ReadToEnd(); - process.WaitForExit(); + if (!process.WaitForExit(waitForExitTimeOut)) + { + result += $"\nERROR: TIMED OUT {waitForExitTimeOut}ms BEFORE COMPLETION"; + } return result; } } - private static string RunOSX(string cmd, string arguments) + private static string RunOSX(string cmd, string arguments, int waitForExitTimeOut = int.MaxValue) { using (var process = new Process()) { @@ -112,7 +118,10 @@ namespace AyaNova.Util process.Start(); string result = process.StandardOutput.ReadToEnd(); - process.WaitForExit(); + if (!process.WaitForExit(waitForExitTimeOut)) + { + result += $"\nERROR: TIMED OUT {waitForExitTimeOut}ms BEFORE COMPLETION"; + } return result; } }