server/AyaNova/biz/JobsBiz.cs

@@ -338,13 +338,20 @@ namespace AyaNova.Biz
                 }
 
                 //STOCK JOBS           
-
+                //Capture metrics  
+                CoreJobMetricsSnapshot.DoJob();
 
 
 
 
                 //Notifications
             }
+            catch (OperationCanceledException e)
+            {
+                log.LogError(e, "JobsBiz::ProcessJobsAsync jobs cancelled likely due to timeout");
+                System.Diagnostics.Debug.WriteLine($"JobsBiz::ProcessJobsAsync {nameof(OperationCanceledException)} thrown with message: {e.Message}");
+                throw;
+            }
             catch (Exception ex)
             {
                 log.LogError(ex, "JobsBiz::ProcessJobsAsync unexpected error during processing");

server/AyaNova/generator/CoreJobBackup.cs

@@ -19,6 +19,7 @@ namespace AyaNova.Biz
     {
         private static ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("CoreJobBackup");
         private static bool BackupIsRunning = false;
+        private const int MAXIMUM_MS_ALLOWED_FOR_PROCESSING = 15 * 60 * 1000;//wild assed guess 15 minutes maximum to run backup command, ever
         ////////////////////////////////////////////////////////////////////////////////////////////////
         // 
         //
@@ -26,18 +27,18 @@ namespace AyaNova.Biz
         {
             //## TEST TEMPORARY
             //This should trigger the kill switch for generator
-             System.Diagnostics.Debug.WriteLine($"CoreJobBackup test wait 5 minutes starting now...");
+            System.Diagnostics.Debug.WriteLine($"CoreJobBackup test wait 2 minutes starting now...");
-            await Task.Delay(new TimeSpan(0,5,0));
+            await Task.Delay(new TimeSpan(0, 2, 0));
 
             if (BackupIsRunning) return;
             if (!OnDemand)
             {
-                log.LogTrace("Checking if backup should run");              
+                log.LogTrace("Checking if backup should run");
                 if (DateTime.UtcNow < ServerGlobalOpsSettingsCache.NextBackup)
                 {
-                    log.LogTrace("Not past backup time yet"); 
+                    log.LogTrace("Not past backup time yet");
                     return;
-                }               
+                }
             }
             AyaNova.Api.ControllerHelpers.ApiServerState apiServerState = null;
             try
@@ -48,7 +49,7 @@ namespace AyaNova.Biz
                 apiServerState.SetClosed("BACKUP RUNNING");
                 await JobsBiz.LogJobAsync(Guid.Empty, $"Starting backup job {(OnDemand ? "manual / on demand" : "scheduled") } ");
                 log.LogDebug("Backup starting");
-                var DemandFileNamePrepend=OnDemand?"manual-":string.Empty;
+                var DemandFileNamePrepend = OnDemand ? "manual-" : string.Empty;
                 //*************
                 //DO DATA BACKUP
                 //build command
@@ -68,16 +69,16 @@ namespace AyaNova.Biz
 
                 var Arguments = $"{DBNameParameter} -Fc > {DataBackupFile}";
 
-                var Result = RunProgram.Run(BackupUtilityCommand, Arguments, log);
+                var Result = RunProgram.Run(BackupUtilityCommand, Arguments, log, MAXIMUM_MS_ALLOWED_FOR_PROCESSING);
                 if (string.IsNullOrWhiteSpace(Result))
                 {
-
                     log.LogDebug("BACKUP SUCCESSFUL (NO ERROR)");
                 }
                 else
                 {
                     await JobsBiz.LogJobAsync(Guid.Empty, $"Error during data backup \"{Result}\"");
                     log.LogError($"BACKUP ERROR: {Result}");
+                    //TODO:OPSNOTIFY
                 }
 
                 //DO FILE BACKUP IF ATTACHMENTS BACKED UP
@@ -100,12 +101,13 @@ namespace AyaNova.Biz
                 await JobsBiz.LogJobAsync(Guid.Empty, "Backup failed with errors:");
                 await JobsBiz.LogJobAsync(Guid.Empty, ExceptionUtil.ExtractAllExceptionMessages(ex));
                 log.LogError(ex, "Backup failed");
+                 //TODO:OPSNOTIFY
                 throw ex;
             }
             finally
             {
                 //bump the backup date if automatic backup
-                if(!OnDemand)
+                if (!OnDemand)
                     ServerGlobalOpsSettingsCache.SetNextBackup();
                 apiServerState.ResumePriorState();
                 BackupIsRunning = false;

server/AyaNova/generator/Generate.cs

@@ -2,8 +2,6 @@ using System.Threading;
 using System;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.DependencyInjection;
-using AyaNova.Models;
 using AyaNova.Api.ControllerHelpers;
 using AyaNova.Biz;
 using AyaNova.Util;

server/AyaNova/util/RunProgram.cs

@@ -10,22 +10,22 @@ namespace AyaNova.Util
     {
 
 
-        public static string Run(string cmd, string arguments, ILogger log = null)
+        public static string Run(string cmd, string arguments, ILogger log = null, int waitForExitTimeOut = int.MaxValue)
         {
             try
             {
                 if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
                 {
-                    return RunWindows(cmd, arguments);
+                    return RunWindows(cmd, arguments, waitForExitTimeOut);
                 }
                 else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
                 {
-                    return RunOSX(cmd, arguments);
+                    return RunOSX(cmd, arguments, waitForExitTimeOut);
 
                 }
                 else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
                 {
-                    return RunLinuxBash(cmd, arguments);
+                    return RunLinuxBash(cmd, arguments, waitForExitTimeOut);
                 }
             }
             catch (Exception ex)
@@ -46,35 +46,38 @@ namespace AyaNova.Util
 
 
 
-        private static string RunWindows(string cmd, string arguments)
+        private static string RunWindows(string cmd, string arguments, int waitForExitTimeOut = int.MaxValue)
         {
-           //RunProgram.Run("cmd.exe",FullRunCommand, log);
+            //RunProgram.Run("cmd.exe",FullRunCommand, log);
-           // var FullRunCommand=$"/C {BackupUtilityCommand} {Arguments}";
+            // var FullRunCommand=$"/C {BackupUtilityCommand} {Arguments}";
-            
+
-//for Windows need to pass to cmd.exe because often have command line piping etc
+            //for Windows need to pass to cmd.exe because often have command line piping etc
-            var args=$"/C {cmd} {arguments}";
+            var args = $"/C {cmd} {arguments}";
             using (var process = new Process())
             {
                 process.StartInfo = new ProcessStartInfo
                 {
                     FileName = "cmd.exe",
-                    Arguments = args,                    
+                    Arguments = args,
                     RedirectStandardOutput = true,
                     RedirectStandardError = true,
                     UseShellExecute = false,
                     CreateNoWindow = true,
                 };
-               
+
 
                 process.Start();
 
                 string result = $"{process.StandardOutput.ReadToEnd()}{process.StandardError.ReadToEnd()} ";
-                process.WaitForExit();
+                if (!process.WaitForExit(waitForExitTimeOut))
+                {
+                    result += $"\nERROR: TIMED OUT {waitForExitTimeOut}ms BEFORE COMPLETION";
+                }
                 return result;
             }
         }
 
-        private static string RunLinuxBash(string cmd, string arguments)
+        private static string RunLinuxBash(string cmd, string arguments, int waitForExitTimeOut = int.MaxValue)
         {
             var escapedArgs = $"{cmd} {arguments}".Replace("\"", "\\\"");
 
@@ -91,13 +94,16 @@ namespace AyaNova.Util
 
                 process.Start();
                 string result = process.StandardOutput.ReadToEnd();
-                process.WaitForExit();
+                if (!process.WaitForExit(waitForExitTimeOut))
+                {
+                    result += $"\nERROR: TIMED OUT {waitForExitTimeOut}ms BEFORE COMPLETION";
+                }
                 return result;
             }
         }
 
 
-        private static string RunOSX(string cmd, string arguments)
+        private static string RunOSX(string cmd, string arguments, int waitForExitTimeOut = int.MaxValue)
         {
             using (var process = new Process())
             {
@@ -112,7 +118,10 @@ namespace AyaNova.Util
 
                 process.Start();
                 string result = process.StandardOutput.ReadToEnd();
-                process.WaitForExit();
+                if (!process.WaitForExit(waitForExitTimeOut))
+                {
+                    result += $"\nERROR: TIMED OUT {waitForExitTimeOut}ms BEFORE COMPLETION";
+                }
                 return result;
             }
         }