admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Browse Source
This commit is contained in:
John Cardinal
2020-05-25 22:36:40 +00:00
parent a10a3e3069
commit 486e7db27c
4 changed files with 41 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
server/AyaNova/biz/JobsBiz.cs
View File

@@ -243,7 +243,7 @@ namespace AyaNova.Biz
/// Process all jobs (stock jobs and those found in operations table) /// Process all jobs (stock jobs and those found in operations table)
/// </summary> /// </summary>
/// <returns></returns> /// <returns></returns>
internal static async Task ProcessJobsAsync(CancellationToken ctoken) internal static async Task ProcessJobsAsync()
{ {
if (ActivelyProcessing) if (ActivelyProcessing)
{ {
@@ -254,11 +254,11 @@ namespace AyaNova.Biz
ActivelyProcessing = true; ActivelyProcessing = true;
try try
{ {
ctoken.ThrowIfCancellationRequested();
//Sweep jobs table //Sweep jobs table
System.Diagnostics.Debug.WriteLine($"JobsBiz processing sweeper"); System.Diagnostics.Debug.WriteLine($"JobsBiz processing sweeper");
await CoreJobSweeper.DoSweepAsync(ctoken);//run exclusively await CoreJobSweeper.DoSweepAsync();//run exclusively
//BIZOBJECT DYNAMIC JOBS //BIZOBJECT DYNAMIC JOBS
//get a list of exclusive jobs that are due to happen //get a list of exclusive jobs that are due to happen
@@ -268,7 +268,6 @@ namespace AyaNova.Biz
{ {
try try
{ {
ctoken.ThrowIfCancellationRequested();
System.Diagnostics.Debug.WriteLine($"JobsBiz processing exclusive biz job {j.Name}"); System.Diagnostics.Debug.WriteLine($"JobsBiz processing exclusive biz job {j.Name}");
await ProcessJobAsync(j); await ProcessJobAsync(j);
} }
@@ -281,7 +280,6 @@ namespace AyaNova.Biz
} }
} }
ctoken.ThrowIfCancellationRequested();
System.Diagnostics.Debug.WriteLine($"JobsBiz processing exclusive license check"); System.Diagnostics.Debug.WriteLine($"JobsBiz processing exclusive license check");
//License check //License check
long CurrentActiveCount = await UserBiz.ActiveCountAsync(); long CurrentActiveCount = await UserBiz.ActiveCountAsync();
@@ -294,7 +292,6 @@ namespace AyaNova.Biz
log.LogCritical(msg); log.LogCritical(msg);
return; return;
} }
ctoken.ThrowIfCancellationRequested();
//backup //backup
System.Diagnostics.Debug.WriteLine($"JobsBiz processing backup"); System.Diagnostics.Debug.WriteLine($"JobsBiz processing backup");
await CoreJobBackup.DoWorkAsync();//sb exclusive await CoreJobBackup.DoWorkAsync();//sb exclusive
@@ -316,16 +313,16 @@ namespace AyaNova.Biz
//This area may turn out to need a re-write in future, but I think it might only involve this block and ProcessJobAsync //This area may turn out to need a re-write in future, but I think it might only involve this block and ProcessJobAsync
//the actual individual objects that are responsible for jobs will likely not need a signature rewrite or anything (I hope) //the actual individual objects that are responsible for jobs will likely not need a signature rewrite or anything (I hope)
//For now I'z hoping that no job will be so slow that it can hold up all the other jobs indefinitely. //For now I'z hoping that no job will be so slow that it can hold up all the other jobs indefinitely.
ctoken.ThrowIfCancellationRequested();
List<OpsJob> sharedJobs = await GetReadyJobsNotExlusiveOnlyAsync(); List<OpsJob> sharedJobs = await GetReadyJobsNotExlusiveOnlyAsync();
foreach (OpsJob j in sharedJobs) foreach (OpsJob j in sharedJobs)
{ {
try try
{ {
ctoken.ThrowIfCancellationRequested();
System.Diagnostics.Debug.WriteLine($"JobsBiz processing NON-exclusive biz job {j.Name}"); System.Diagnostics.Debug.WriteLine($"JobsBiz processing NON-exclusive biz job {j.Name}");
// Task.Run(() => FireAway()); // Task.Run(() => FireAway());
Task.Run(() => ProcessJobAsync(j), ctoken); TaskUtil.Forget(Task.Run(() => ProcessJobAsync(j)));
// await ProcessJobAsync(j, ct); // await ProcessJobAsync(j, ct);
} }
catch (Exception ex) catch (Exception ex)

20
server/AyaNova/generator/CoreJobSweeper.cs
View File

@@ -28,10 +28,10 @@ namespace AyaNova.Biz
//////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////
// DoSweep // DoSweep
// //
public static async Task DoSweepAsync(CancellationToken ctoken) public static async Task DoSweepAsync()
{ {
ctoken.ThrowIfCancellationRequested();
//This will get triggered roughly every minute, but we don't want to sweep that frequently //This will get triggered roughly every minute, but we don't want to sweep that frequently
if (DateTime.UtcNow - lastSweep < SWEEP_EVERY_INTERVAL) if (DateTime.UtcNow - lastSweep < SWEEP_EVERY_INTERVAL)
return; return;
@@ -43,32 +43,32 @@ namespace AyaNova.Biz
//SWEEP SUCCESSFUL JOBS //SWEEP SUCCESSFUL JOBS
//calculate cutoff to delete //calculate cutoff to delete
DateTime dtDeleteCutoff = DateTime.UtcNow - SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN; DateTime dtDeleteCutoff = DateTime.UtcNow - SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Completed, ctoken); await sweepAsync(ct, dtDeleteCutoff, JobStatus.Completed);
ctoken.ThrowIfCancellationRequested();
//SWEEP FAILED JOBS //SWEEP FAILED JOBS
//calculate cutoff to delete //calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN; dtDeleteCutoff = DateTime.UtcNow - FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Failed, ctoken); await sweepAsync(ct, dtDeleteCutoff, JobStatus.Failed);
ctoken.ThrowIfCancellationRequested();
//KILL STUCK JOBS //KILL STUCK JOBS
//calculate cutoff to delete //calculate cutoff to delete
DateTime dtRunningDeadline = DateTime.UtcNow - RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN; DateTime dtRunningDeadline = DateTime.UtcNow - RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN;
await killStuckJobsAsync(ct, dtRunningDeadline); await killStuckJobsAsync(ct, dtRunningDeadline);
ctoken.ThrowIfCancellationRequested();
//SWEEP INTERNAL JOB LOG //SWEEP INTERNAL JOB LOG
//calculate cutoff to delete //calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - INTERNAL_JOBS_LOGS_DELETE_AFTER_THIS_TIMESPAN; dtDeleteCutoff = DateTime.UtcNow - INTERNAL_JOBS_LOGS_DELETE_AFTER_THIS_TIMESPAN;
await SweepInternalJobsLogsAsync(ct, dtDeleteCutoff); await SweepInternalJobsLogsAsync(ct, dtDeleteCutoff);
ctoken.ThrowIfCancellationRequested();
} }
lastSweep = DateTime.UtcNow; lastSweep = DateTime.UtcNow;
} }
private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus, CancellationToken ctoken)//AyContext ct, private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus)
{ {
// AyContext ct = ServiceProviderProvider.DBContext; // AyContext ct = ServiceProviderProvider.DBContext;
//Get the deleteable succeeded jobs list //Get the deleteable succeeded jobs list
@@ -84,7 +84,7 @@ namespace AyaNova.Biz
{ {
try try
{ {
ctoken.ThrowIfCancellationRequested();
await JobsBiz.RemoveJobAndLogsAsync(j.GId); await JobsBiz.RemoveJobAndLogsAsync(j.GId);
} }
catch (Exception ex) catch (Exception ex)

4
server/AyaNova/generator/Generate.cs
View File

@@ -20,7 +20,7 @@ namespace AyaNova.Generator
public class GeneratorService : BackgroundService public class GeneratorService : BackgroundService
{ {
private readonly ILogger<GeneratorService> log; private readonly ILogger<GeneratorService> log;
private const int MAXIMUM_MS_ALLOWED_FOR_PROCESSING_ALL_JOBS = 1 * 60 * 1000;//1 minutes TEST TEST TEST ##### // private const int MAXIMUM_MS_ALLOWED_FOR_PROCESSING_ALL_JOBS = 1 * 60 * 1000;//1 minutes TEST TEST TEST #####
#if(DEBUG) #if(DEBUG)
private const int GENERATE_SECONDS = 5; private const int GENERATE_SECONDS = 5;
#else #else
@@ -65,7 +65,7 @@ namespace AyaNova.Generator
//Capture metrics //Capture metrics
CoreJobMetricsSnapshot.DoJob(); CoreJobMetricsSnapshot.DoJob();
//TODO: this should be big timeout and then inside the process jobs each job has it's own timeout //TODO: this should be big timeout and then inside the process jobs each job has it's own timeout
await TaskUtil.WithTimeoutAfterStart(ctoken => JobsBiz.ProcessJobsAsync(ctoken), TimeSpan.FromMilliseconds(MAXIMUM_MS_ALLOWED_FOR_PROCESSING_ALL_JOBS)); await JobsBiz.ProcessJobsAsync();
System.Diagnostics.Debug.WriteLine($"### GENERATE BACK FROM calling JobsBiz.ProcessJobs"); System.Diagnostics.Debug.WriteLine($"### GENERATE BACK FROM calling JobsBiz.ProcessJobs");
} }
} }

24
server/AyaNova/util/TaskUtil.cs
View File

@@ -1,4 +1,5 @@
using System; using System;
using System.Linq;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
@@ -8,7 +9,9 @@ namespace AyaNova.Util
{ {
///<summary> ///<summary>
///Usage: ///Usage:
///await WithTimeoutAfterStart(ct => SomeOperationAsync(ct), TimeSpan.FromMilliseconds(n)); ///await WithTimeoutAfterStart(ctoken => SomeOperationAsync(ctoken), TimeSpan.FromMilliseconds(n));
///in callee call this regularly ctoken.ThrowIfCancellationRequested();
///and pass the ctoken into any time consuming system methods called in turn
///</summary> ///</summary>
public static async Task WithTimeoutAfterStart(Func<CancellationToken, Task> operation, TimeSpan timeout) public static async Task WithTimeoutAfterStart(Func<CancellationToken, Task> operation, TimeSpan timeout)
{ {
@@ -19,6 +22,25 @@ namespace AyaNova.Util
await task; await task;
} }
///<summary>
///fire and forget a task but bubble up any exceptions with optional ignore some
///</summary>
public static async void Forget(this Task task, params Type[] acceptableExceptions)
{
//https://stackoverflow.com/a/22864616/8939
try
{
await task.ConfigureAwait(false);
}
catch (Exception ex)
{
// TODO: consider whether derived types are also acceptable.
if (!acceptableExceptions.Contains(ex.GetType()))
throw;
}
}
}//eoc }//eoc
}//eons }//eons