diff --git a/server/AyaNova/biz/JobsBiz.cs b/server/AyaNova/biz/JobsBiz.cs index f16b6c6d..b723b42d 100644 --- a/server/AyaNova/biz/JobsBiz.cs +++ b/server/AyaNova/biz/JobsBiz.cs @@ -243,7 +243,7 @@ namespace AyaNova.Biz /// Process all jobs (stock jobs and those found in operations table) /// /// - internal static async Task ProcessJobsAsync(CancellationToken ctoken) + internal static async Task ProcessJobsAsync() { if (ActivelyProcessing) { @@ -254,11 +254,11 @@ namespace AyaNova.Biz ActivelyProcessing = true; try { - ctoken.ThrowIfCancellationRequested(); + //Sweep jobs table System.Diagnostics.Debug.WriteLine($"JobsBiz processing sweeper"); - await CoreJobSweeper.DoSweepAsync(ctoken);//run exclusively + await CoreJobSweeper.DoSweepAsync();//run exclusively //BIZOBJECT DYNAMIC JOBS //get a list of exclusive jobs that are due to happen @@ -268,7 +268,6 @@ namespace AyaNova.Biz { try { - ctoken.ThrowIfCancellationRequested(); System.Diagnostics.Debug.WriteLine($"JobsBiz processing exclusive biz job {j.Name}"); await ProcessJobAsync(j); } @@ -281,7 +280,6 @@ namespace AyaNova.Biz } } - ctoken.ThrowIfCancellationRequested(); System.Diagnostics.Debug.WriteLine($"JobsBiz processing exclusive license check"); //License check long CurrentActiveCount = await UserBiz.ActiveCountAsync(); @@ -294,7 +292,6 @@ namespace AyaNova.Biz log.LogCritical(msg); return; } - ctoken.ThrowIfCancellationRequested(); //backup System.Diagnostics.Debug.WriteLine($"JobsBiz processing backup"); await CoreJobBackup.DoWorkAsync();//sb exclusive @@ -316,16 +313,16 @@ namespace AyaNova.Biz //This area may turn out to need a re-write in future, but I think it might only involve this block and ProcessJobAsync //the actual individual objects that are responsible for jobs will likely not need a signature rewrite or anything (I hope) //For now I'z hoping that no job will be so slow that it can hold up all the other jobs indefinitely. - ctoken.ThrowIfCancellationRequested(); + List sharedJobs = await GetReadyJobsNotExlusiveOnlyAsync(); foreach (OpsJob j in sharedJobs) { try { - ctoken.ThrowIfCancellationRequested(); + System.Diagnostics.Debug.WriteLine($"JobsBiz processing NON-exclusive biz job {j.Name}"); // Task.Run(() => FireAway()); - Task.Run(() => ProcessJobAsync(j), ctoken); + TaskUtil.Forget(Task.Run(() => ProcessJobAsync(j))); // await ProcessJobAsync(j, ct); } catch (Exception ex) diff --git a/server/AyaNova/generator/CoreJobSweeper.cs b/server/AyaNova/generator/CoreJobSweeper.cs index 0db9eaf7..f76ce747 100644 --- a/server/AyaNova/generator/CoreJobSweeper.cs +++ b/server/AyaNova/generator/CoreJobSweeper.cs @@ -28,10 +28,10 @@ namespace AyaNova.Biz //////////////////////////////////////////////////////////////////////////////////////////////// // DoSweep // - public static async Task DoSweepAsync(CancellationToken ctoken) + public static async Task DoSweepAsync() { - ctoken.ThrowIfCancellationRequested(); + //This will get triggered roughly every minute, but we don't want to sweep that frequently if (DateTime.UtcNow - lastSweep < SWEEP_EVERY_INTERVAL) return; @@ -43,32 +43,32 @@ namespace AyaNova.Biz //SWEEP SUCCESSFUL JOBS //calculate cutoff to delete DateTime dtDeleteCutoff = DateTime.UtcNow - SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN; - await sweepAsync(ct, dtDeleteCutoff, JobStatus.Completed, ctoken); + await sweepAsync(ct, dtDeleteCutoff, JobStatus.Completed); - ctoken.ThrowIfCancellationRequested(); + //SWEEP FAILED JOBS //calculate cutoff to delete dtDeleteCutoff = DateTime.UtcNow - FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN; - await sweepAsync(ct, dtDeleteCutoff, JobStatus.Failed, ctoken); + await sweepAsync(ct, dtDeleteCutoff, JobStatus.Failed); - ctoken.ThrowIfCancellationRequested(); + //KILL STUCK JOBS //calculate cutoff to delete DateTime dtRunningDeadline = DateTime.UtcNow - RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN; await killStuckJobsAsync(ct, dtRunningDeadline); - ctoken.ThrowIfCancellationRequested(); + //SWEEP INTERNAL JOB LOG //calculate cutoff to delete dtDeleteCutoff = DateTime.UtcNow - INTERNAL_JOBS_LOGS_DELETE_AFTER_THIS_TIMESPAN; await SweepInternalJobsLogsAsync(ct, dtDeleteCutoff); - ctoken.ThrowIfCancellationRequested(); + } lastSweep = DateTime.UtcNow; } - private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus, CancellationToken ctoken)//AyContext ct, + private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus) { // AyContext ct = ServiceProviderProvider.DBContext; //Get the deleteable succeeded jobs list @@ -84,7 +84,7 @@ namespace AyaNova.Biz { try { - ctoken.ThrowIfCancellationRequested(); + await JobsBiz.RemoveJobAndLogsAsync(j.GId); } catch (Exception ex) diff --git a/server/AyaNova/generator/Generate.cs b/server/AyaNova/generator/Generate.cs index 44b2c23c..4b252801 100644 --- a/server/AyaNova/generator/Generate.cs +++ b/server/AyaNova/generator/Generate.cs @@ -20,7 +20,7 @@ namespace AyaNova.Generator public class GeneratorService : BackgroundService { private readonly ILogger log; - private const int MAXIMUM_MS_ALLOWED_FOR_PROCESSING_ALL_JOBS = 1 * 60 * 1000;//1 minutes TEST TEST TEST ##### + // private const int MAXIMUM_MS_ALLOWED_FOR_PROCESSING_ALL_JOBS = 1 * 60 * 1000;//1 minutes TEST TEST TEST ##### #if(DEBUG) private const int GENERATE_SECONDS = 5; #else @@ -65,7 +65,7 @@ namespace AyaNova.Generator //Capture metrics CoreJobMetricsSnapshot.DoJob(); //TODO: this should be big timeout and then inside the process jobs each job has it's own timeout - await TaskUtil.WithTimeoutAfterStart(ctoken => JobsBiz.ProcessJobsAsync(ctoken), TimeSpan.FromMilliseconds(MAXIMUM_MS_ALLOWED_FOR_PROCESSING_ALL_JOBS)); + await JobsBiz.ProcessJobsAsync(); System.Diagnostics.Debug.WriteLine($"### GENERATE BACK FROM calling JobsBiz.ProcessJobs"); } } diff --git a/server/AyaNova/util/TaskUtil.cs b/server/AyaNova/util/TaskUtil.cs index 72533501..d148b87a 100644 --- a/server/AyaNova/util/TaskUtil.cs +++ b/server/AyaNova/util/TaskUtil.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -8,7 +9,9 @@ namespace AyaNova.Util { /// ///Usage: - ///await WithTimeoutAfterStart(ct => SomeOperationAsync(ct), TimeSpan.FromMilliseconds(n)); + ///await WithTimeoutAfterStart(ctoken => SomeOperationAsync(ctoken), TimeSpan.FromMilliseconds(n)); + ///in callee call this regularly ctoken.ThrowIfCancellationRequested(); + ///and pass the ctoken into any time consuming system methods called in turn /// public static async Task WithTimeoutAfterStart(Func operation, TimeSpan timeout) { @@ -19,6 +22,25 @@ namespace AyaNova.Util await task; } + + /// + ///fire and forget a task but bubble up any exceptions with optional ignore some + /// + public static async void Forget(this Task task, params Type[] acceptableExceptions) + { + //https://stackoverflow.com/a/22864616/8939 + try + { + await task.ConfigureAwait(false); + } + catch (Exception ex) + { + // TODO: consider whether derived types are also acceptable. + if (!acceptableExceptions.Contains(ex.GetType())) + throw; + } + } + }//eoc }//eons \ No newline at end of file