diff --git a/server/AyaNova/Controllers/BackupController.cs b/server/AyaNova/Controllers/BackupController.cs
index 07f6dc29..f2a0fdac 100644
--- a/server/AyaNova/Controllers/BackupController.cs
+++ b/server/AyaNova/Controllers/BackupController.cs
@@ -68,7 +68,7 @@ namespace AyaNova.Api.Controllers
j.JobType = JobType.Backup;
j.SubType = JobSubType.NotSet;
j.Exclusive = true;
- await JobsBiz.AddJobAsync(j, ct);
+ await JobsBiz.AddJobAsync(j);
await EventLogProcessor.LogEventToDatabaseAsync(new Event(UserIdFromContext.Id(HttpContext.Items), 0, AyaType.ServerJob, AyaEvent.Created, JobName), ct);
return Accepted(new { JobId = j.GId });
}
diff --git a/server/AyaNova/biz/BizObjectFactory.cs b/server/AyaNova/biz/BizObjectFactory.cs
index 8f752f86..50b2cb92 100644
--- a/server/AyaNova/biz/BizObjectFactory.cs
+++ b/server/AyaNova/biz/BizObjectFactory.cs
@@ -20,8 +20,9 @@ namespace AyaNova.Biz
//Returns the biz object class that corresponds to the type presented
//Used by SEARCH and objects with JOBS
- internal static BizObject GetBizObject(AyaType ayaType, AyContext dbcontext, long userId = 1, AuthorizationRoles roles = AuthorizationRoles.All)
+ internal static BizObject GetBizObject(AyaType ayaType, long userId = 1, AuthorizationRoles roles = AuthorizationRoles.All)
{
+ var dbcontext = ServiceProviderProvider.DBContext;
switch (ayaType)
{
//CoreBizObject add here
diff --git a/server/AyaNova/biz/JobsBiz.cs b/server/AyaNova/biz/JobsBiz.cs
index 8cfbc6b0..f592c340 100644
--- a/server/AyaNova/biz/JobsBiz.cs
+++ b/server/AyaNova/biz/JobsBiz.cs
@@ -1,6 +1,7 @@
using System;
using System.Linq;
using System.Threading.Tasks;
+using System.Threading;
using System.Collections.Generic;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
@@ -22,10 +23,11 @@ namespace AyaNova.Biz
/// Get a non tracked list of jobs for an object
///
///
- ///
+ ///
///
- internal static async Task> GetJobsForObjectAsync(AyaTypeId ayObj, AyContext ct)
+ internal static async Task> GetJobsForObjectAsync(AyaTypeId ayObj)
{
+ var ct = ServiceProviderProvider.DBContext;
return await ct.OpsJob
.AsNoTracking()
.Where(z => z.ObjectId == ayObj.ObjectId && z.ObjectType == ayObj.ObjectType)
@@ -39,9 +41,9 @@ namespace AyaNova.Biz
/// Get a non tracked list of jobs that are ready to process and exclusive only
///
///
- internal static async Task> GetReadyJobsExclusiveOnlyAsync(AyContext ct)
+ internal static async Task> GetReadyJobsExclusiveOnlyAsync()
{
- return await GetReadyJobsAsync(true, ct);
+ return await GetReadyJobsAsync(true);
}
@@ -49,9 +51,9 @@ namespace AyaNova.Biz
/// Get a non tracked list of jobs that are ready to process and exclusive only
///
///
- internal static async Task> GetReadyJobsNotExlusiveOnlyAsync(AyContext ct)
+ internal static async Task> GetReadyJobsNotExlusiveOnlyAsync()
{
- return await GetReadyJobsAsync(false, ct);
+ return await GetReadyJobsAsync(false);
}
@@ -59,8 +61,9 @@ namespace AyaNova.Biz
/// Get a non tracked list of jobs filtered by exclusivity
///
///
- private static async Task> GetReadyJobsAsync(bool exclusiveOnly, AyContext ct)
+ private static async Task> GetReadyJobsAsync(bool exclusiveOnly)
{
+ var ct = ServiceProviderProvider.DBContext;
var ret = await ct.OpsJob
.AsNoTracking()
.Where(z => z.StartAfter < System.DateTime.UtcNow && z.Exclusive == exclusiveOnly && z.JobStatus == JobStatus.Sleeping)
@@ -76,8 +79,9 @@ namespace AyaNova.Biz
/// could be running or sleeping
///
///
- internal static async Task> GetAllSleepingOrRunningJobsAsync(AyContext ct)
+ internal static async Task> GetAllSleepingOrRunningJobsAsync()
{
+ var ct = ServiceProviderProvider.DBContext;
var ret = await ct.OpsJob
.AsNoTracking()
.Where(z => z.JobStatus == JobStatus.Sleeping || z.JobStatus == JobStatus.Running)
@@ -92,8 +96,9 @@ namespace AyaNova.Biz
/// Get a non tracked list of all jobs for a JobType
///
///
- internal static async Task> GetAllJobsForJobTypeAsync(AyContext ct, JobType jobType)
+ internal static async Task> GetAllJobsForJobTypeAsync(JobType jobType)
{
+ var ct = ServiceProviderProvider.DBContext;
var ret = await ct.OpsJob
.AsNoTracking()
.Where(z => z.JobType == jobType)
@@ -108,8 +113,9 @@ namespace AyaNova.Biz
/// Get a non tracked list of all jobs that are status running but have no last activity for XX HOURS
///
///
- internal static async Task> GetPotentiallyDeadRunningJobsAsync(AyContext ct)
+ internal static async Task> GetPotentiallyDeadRunningJobsAsync()
{
+ var ct = ServiceProviderProvider.DBContext;
var ret = await ct.OpsJob
.AsNoTracking()
.Where(z => z.JobStatus == JobStatus.Sleeping || z.JobStatus == JobStatus.Running)
@@ -124,8 +130,9 @@ namespace AyaNova.Biz
/// Get a count of all jobs for a JobStatus
///
///
- internal static async Task GetCountForJobStatusAsync(AyContext ct, JobStatus jobStatus)
+ internal static async Task GetCountForJobStatusAsync(JobStatus jobStatus)
{
+ var ct = ServiceProviderProvider.DBContext;
var ret = await ct.OpsJob
.Where(z => z.JobStatus == jobStatus)
.LongCountAsync();
@@ -138,10 +145,11 @@ namespace AyaNova.Biz
/// Add a new job to the database
///
///
- ///
+
///
- internal static async Task AddJobAsync(OpsJob newJob, AyContext ct)
+ internal static async Task AddJobAsync(OpsJob newJob)
{
+ var ct = ServiceProviderProvider.DBContext;
await ct.OpsJob.AddAsync(newJob);
await ct.SaveChangesAsync();
return newJob;
@@ -151,110 +159,57 @@ namespace AyaNova.Biz
/// Remove any jobs or logs for the object in question
///
///
- ///
+ ///
internal static async Task DeleteJobsForObjectAsync(AyaTypeId ayObj, AyContext ct)
{
//Get a list of all jobid's for the object passed in
- List jobsForObject = await GetJobsForObjectAsync(ayObj, ct);
-
- //short circuit
+ List jobsForObject = await GetJobsForObjectAsync(ayObj);
if (jobsForObject.Count == 0)
return;
-
- using (var transaction = await ct.Database.BeginTransactionAsync())
+ foreach (OpsJob jobToBeDeleted in jobsForObject)
{
- try
- {
- foreach (OpsJob jobToBeDeleted in jobsForObject)
- {
- await RemoveJobAndLogsAsync(ct, jobToBeDeleted.GId);
- }
-
- // Commit transaction if all commands succeed, transaction will auto-rollback
- // when disposed if either commands fails
- await transaction.CommitAsync();
- }
- catch (Exception ex)
- {
- throw ex;
- }
+ await RemoveJobAndLogsAsync(jobToBeDeleted.GId);
}
}
-
-
- ///
- /// Remove job and logs for that job
- ///
- ///
- ///
- internal static async Task DeleteJobAndLogAsync(Guid jobId, AyContext ct)
- {
- using (var transaction = await ct.Database.BeginTransactionAsync())
- {
- try
- {
- await RemoveJobAndLogsAsync(ct, jobId);
- // Commit transaction if all commands succeed, transaction will auto-rollback
- // when disposed if either commands fails
- await transaction.CommitAsync();
- }
- catch (Exception ex)
- {
- throw ex;
- }
- }
- }
-
-
-
///
/// REmove the job and it's logs
- ///
- ///
+ ///
///
- private static async Task RemoveJobAndLogsAsync(AyContext ct, Guid jobIdToBeDeleted)
+ internal static async Task RemoveJobAndLogsAsync(Guid jobIdToBeDeleted)
{
- // //delete logs
- // await ct.Database.ExecuteSqlCommandAsync("delete from aopsjoblog where jobid = {0}", new object[] { jobIdToBeDeleted });
-
- // //delete the job
- // await ct.Database.ExecuteSqlCommandAsync("delete from aopsjob where gid = {0}", new object[] { jobIdToBeDeleted });
-
- //delete logs
- await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjoblog where jobid = {jobIdToBeDeleted}");
-
- //delete the job
- await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjob where gid = {jobIdToBeDeleted}");
+ var ct = ServiceProviderProvider.DBContext;
+ using (var transaction = await ct.Database.BeginTransactionAsync())
+ {
+ try
+ {
+ //delete logs
+ await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjoblog where jobid = {jobIdToBeDeleted}");
+ //delete the job
+ await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjob where gid = {jobIdToBeDeleted}");
+ // Commit transaction if all commands succeed, transaction will auto-rollback
+ // when disposed if either commands fails
+ await transaction.CommitAsync();
+ }
+ catch (Exception ex)
+ {
+ throw ex;
+ }
+ }
}
-
-
-
///
/// Make a log entry for a job
- /// (no context version)
///
/// (NOTE: Guid.empty indicates internal job)
///
+ ///
internal static async Task LogJobAsync(Guid jobId, string statusText)
{
- return await LogJobAsync(jobId, statusText, null);
- }
-
- ///
- /// Make a log entry for a job
- ///
- /// (NOTE: Guid.empty indicates internal job)
- ///
- ///
- internal static async Task LogJobAsync(Guid jobId, string statusText, AyContext ct)
- {
- if (ct == null)
- ct = ServiceProviderProvider.DBContext;
+ var ct = ServiceProviderProvider.DBContext;
if (string.IsNullOrWhiteSpace(statusText))
statusText = "No status provided";
OpsJobLog newObj = new OpsJobLog();
@@ -270,10 +225,10 @@ namespace AyaNova.Biz
/// Update the status of a job
///
///
- ///
- ///
- internal static async Task UpdateJobStatusAsync(Guid jobId, JobStatus newStatus, AyContext ct)
+ ///
+ internal static async Task UpdateJobStatusAsync(Guid jobId, JobStatus newStatus)
{
+ var ct = ServiceProviderProvider.DBContext;
var oFromDb = await ct.OpsJob.SingleOrDefaultAsync(z => z.GId == jobId);
if (oFromDb == null) return null;
oFromDb.JobStatus = newStatus;
@@ -289,37 +244,70 @@ namespace AyaNova.Biz
/// Process all jobs (stock jobs and those found in operations table)
///
///
- internal static async Task ProcessJobsAsync(AyContext ct, AyaNova.Api.ControllerHelpers.ApiServerState serverState)
+ internal static async Task ProcessJobsAsync(CancellationToken ctoken)
{
-
if (ActivelyProcessing)
{
+ System.Diagnostics.Debug.WriteLine("ProcessJobs called but actively processing other jobs so returning");
log.LogTrace("ProcessJobs called but actively processing other jobs so returning");
return;
}
ActivelyProcessing = true;
try
{
-
+ ctoken.ThrowIfCancellationRequested();
+
+ //Sweep jobs table
+ System.Diagnostics.Debug.WriteLine($"JobsBiz processing sweeper");
+ await CoreJobSweeper.DoSweepAsync(ctoken);//run exclusively
+
//BIZOBJECT DYNAMIC JOBS
//get a list of exclusive jobs that are due to happen
//Call into each item in turn
- List exclusiveJobs = await GetReadyJobsExclusiveOnlyAsync(ct);
+ List exclusiveJobs = await GetReadyJobsExclusiveOnlyAsync();
foreach (OpsJob j in exclusiveJobs)
{
try
{
- await ProcessJobAsync(j, ct);
+ ctoken.ThrowIfCancellationRequested();
+ System.Diagnostics.Debug.WriteLine($"JobsBiz processing exclusive biz job {j.Name}");
+ await ProcessJobAsync(j);
}
catch (Exception ex)
{
log.LogError(ex, $"ProcessJobs::Exclusive -> job {j.Name} failed with exception");
- await LogJobAsync(j.GId, "Job failed with errors:", ct);
- await LogJobAsync(j.GId, ExceptionUtil.ExtractAllExceptionMessages(ex), ct);
- await UpdateJobStatusAsync(j.GId, JobStatus.Failed, ct);
+ await LogJobAsync(j.GId, "Job failed with errors:");
+ await LogJobAsync(j.GId, ExceptionUtil.ExtractAllExceptionMessages(ex));
+ await UpdateJobStatusAsync(j.GId, JobStatus.Failed);
}
}
- //Get a list of non-exlusive jobs that are due
+
+ ctoken.ThrowIfCancellationRequested();
+ System.Diagnostics.Debug.WriteLine($"JobsBiz processing exclusive license check");
+ //License check
+ long CurrentActiveCount = await UserBiz.ActiveCountAsync();
+ long LicensedUserCount = AyaNova.Core.License.ActiveKey.ActiveNumber;
+ // log.LogInformation("JobsBiz::Checking license active count");
+ if (CurrentActiveCount > LicensedUserCount)
+ {
+ var msg = $"E1020 - Active count exceeded capacity";
+ ServiceProviderProvider.ServerState.SetSystemLock(msg);
+ log.LogCritical(msg);
+ return;
+ }
+ ctoken.ThrowIfCancellationRequested();
+ //backup
+ System.Diagnostics.Debug.WriteLine($"JobsBiz processing backup");
+ await CoreJobBackup.DoWorkAsync();//sb exclusive
+ System.Diagnostics.Debug.WriteLine($"JobsBiz processing metrics snapshotter");
+
+
+ ///////////////////////////////////////
+ //NON-EXCLUSIVE JOBS
+ //
+
+
+
//LOOKAT: Parallelize / background this block
//http://www.dotnetcurry.com/dotnet/1360/concurrent-programming-dotnet-core
@@ -329,42 +317,31 @@ namespace AyaNova.Biz
//This area may turn out to need a re-write in future, but I think it might only involve this block and ProcessJobAsync
//the actual individual objects that are responsible for jobs will likely not need a signature rewrite or anything (I hope)
//For now I'z hoping that no job will be so slow that it can hold up all the other jobs indefinitely.
-
- List sharedJobs = await GetReadyJobsNotExlusiveOnlyAsync(ct);
+ ctoken.ThrowIfCancellationRequested();
+ List sharedJobs = await GetReadyJobsNotExlusiveOnlyAsync();
foreach (OpsJob j in sharedJobs)
{
try
{
- await ProcessJobAsync(j, ct);
+ ctoken.ThrowIfCancellationRequested();
+ System.Diagnostics.Debug.WriteLine($"JobsBiz processing NON-exclusive biz job {j.Name}");
+ // Task.Run(() => FireAway());
+ Task.Run(() => ProcessJobAsync(j), ctoken);
+ // await ProcessJobAsync(j, ct);
}
catch (Exception ex)
{
log.LogError(ex, $"ProcessJobs::Shared -> job {j.Name} failed with exception");
- await LogJobAsync(j.GId, "Job failed with errors:", ct);
- await LogJobAsync(j.GId, ExceptionUtil.ExtractAllExceptionMessages(ex), ct);
- await UpdateJobStatusAsync(j.GId, JobStatus.Failed, ct);
+ await LogJobAsync(j.GId, "Job failed with errors:");
+ await LogJobAsync(j.GId, ExceptionUtil.ExtractAllExceptionMessages(ex));
+ await UpdateJobStatusAsync(j.GId, JobStatus.Failed);
}
}
- //STOCK JOBS
- //Sweep jobs table
- await CoreJobSweeper.DoSweepAsync(ct);
- //License check
- long CurrentActiveCount = await UserBiz.ActiveCountAsync();
- long LicensedUserCount = AyaNova.Core.License.ActiveKey.ActiveNumber;
- // log.LogInformation("JobsBiz::Checking license active count");
- if (CurrentActiveCount > LicensedUserCount)
- {
- var msg = $"E1020 - Active count exceeded capacity";
- serverState.SetSystemLock(msg);
- log.LogCritical(msg);
- return;
- }
+ //STOCK JOBS
+
- //backup
- await CoreJobBackup.DoWorkAsync(ct);
- await CoreJobMetricsSnapshot.DoJobAsync(ct);
//Notifications
@@ -377,6 +354,7 @@ namespace AyaNova.Biz
finally
{
ActivelyProcessing = false;
+ System.Diagnostics.Debug.WriteLine($"JobsBiz in Finally - completed run");
}
}
@@ -384,15 +362,14 @@ namespace AyaNova.Biz
///
/// Process a job by calling into it's biz object
///
- ///
- ///
+ ///
///
- internal static async Task ProcessJobAsync(OpsJob job, AyContext ct)
+ internal static async Task ProcessJobAsync(OpsJob job)
{
var JobDescription = $"{job.Name} {job.JobType.ToString()}";
if (job.SubType != JobSubType.NotSet)
JobDescription += $":{job.SubType}";
- await LogJobAsync(job.GId, $"Process job \"{JobDescription}\"", ct);
+ await LogJobAsync(job.GId, $"Process job \"{JobDescription}\"");
log.LogDebug($"ProcessJobAsync -> Processing job {JobDescription}");
IJobObject o = null;
@@ -400,19 +377,19 @@ namespace AyaNova.Biz
{
case JobType.Backup:
//This is called when on demand only, normal backups are processed above with normal system jobs
- await CoreJobBackup.DoWorkAsync(ct, true);
- await UpdateJobStatusAsync(job.GId, JobStatus.Completed, ct);
+ await CoreJobBackup.DoWorkAsync(true);
+ await UpdateJobStatusAsync(job.GId, JobStatus.Completed);
break;
case JobType.TestJob:
- o = (IJobObject)BizObjectFactory.GetBizObject(AyaType.ServerJob, ct);
+ o = (IJobObject)BizObjectFactory.GetBizObject(AyaType.ServerJob);
break;
case JobType.SeedTestData:
- o = (IJobObject)BizObjectFactory.GetBizObject(AyaType.TrialSeeder, ct);
+ o = (IJobObject)BizObjectFactory.GetBizObject(AyaType.TrialSeeder);
break;
case JobType.BulkCoreBizObjectOperation:
//bulk op, hand off to biz object to deal with
//note, convention is that there is an idList in job.jobinfo json if preselected else it's all objects of type
- o = (IJobObject)BizObjectFactory.GetBizObject(job.ObjectType, ct);
+ o = (IJobObject)BizObjectFactory.GetBizObject(job.ObjectType);
break;
default:
throw new System.NotSupportedException($"ProcessJobAsync type {job.JobType.ToString()} is not supported");
diff --git a/server/AyaNova/biz/Search.cs b/server/AyaNova/biz/Search.cs
index 5a64ee32..630e2be7 100644
--- a/server/AyaNova/biz/Search.cs
+++ b/server/AyaNova/biz/Search.cs
@@ -246,8 +246,7 @@ namespace AyaNova.Biz
#endregion dosearch
#region Get info (excerpt)
- public static async Task GetInfoAsync(AyContext ct, long translationId,
- AuthorizationRoles currentUserRoles, long userId, string phrase, int max, AyaType ayaType, long id)
+ public static async Task GetInfoAsync(long translationId, AuthorizationRoles currentUserRoles, long userId, string phrase, int max, AyaType ayaType, long id)
{
//escape literal percentage signs first just in case they are searching for 50% off or something
//https://www.postgresql.org/docs/current/functions-matching.html#FUNCTIONS-LIKE
@@ -262,7 +261,7 @@ namespace AyaNova.Biz
PhraseItems.ToArray();
//get text
- ISearchAbleObject o = (ISearchAbleObject)BizObjectFactory.GetBizObject(ayaType, ct, userId, currentUserRoles);
+ ISearchAbleObject o = (ISearchAbleObject)BizObjectFactory.GetBizObject(ayaType, userId, currentUserRoles);
//get extract
var searchParams = await o.GetSearchResultSummary(id);
diff --git a/server/AyaNova/biz/WorkOrderBiz.cs b/server/AyaNova/biz/WorkOrderBiz.cs
index c4719ddc..54bd63d2 100644
--- a/server/AyaNova/biz/WorkOrderBiz.cs
+++ b/server/AyaNova/biz/WorkOrderBiz.cs
@@ -2441,8 +2441,8 @@ namespace AyaNova.Biz
private async Task ProcessBulkJobAsync(OpsJob job)
{
- await JobsBiz.UpdateJobStatusAsync(job.GId, JobStatus.Running, ct);
- await JobsBiz.LogJobAsync(job.GId, $"Bulk job {job.SubType} started...", ct);
+ await JobsBiz.UpdateJobStatusAsync(job.GId, JobStatus.Running);
+ await JobsBiz.LogJobAsync(job.GId, $"Bulk job {job.SubType} started...");
List idList = new List();
long ProcessedObjectCount = 0;
JObject jobData = JObject.Parse(job.JobInfo);
@@ -2475,7 +2475,7 @@ namespace AyaNova.Biz
{
o = await PutWorkOrderGraphItem(job.ObjectType, o);
if (o == null)
- await JobsBiz.LogJobAsync(job.GId, $"Error processing item {id}: {GetErrorsAsString()}", ct);
+ await JobsBiz.LogJobAsync(job.GId, $"Error processing item {id}: {GetErrorsAsString()}");
else
ProcessedObjectCount++;
}
@@ -2484,12 +2484,12 @@ namespace AyaNova.Biz
}
catch (Exception ex)
{
- await JobsBiz.LogJobAsync(job.GId, $"Error processing item {id}", ct);
- await JobsBiz.LogJobAsync(job.GId, ExceptionUtil.ExtractAllExceptionMessages(ex), ct);
+ await JobsBiz.LogJobAsync(job.GId, $"Error processing item {id}");
+ await JobsBiz.LogJobAsync(job.GId, ExceptionUtil.ExtractAllExceptionMessages(ex));
}
}
- await JobsBiz.LogJobAsync(job.GId, $"Bulk job {job.SubType} processed {ProcessedObjectCount} of {idList.Count}", ct);
- await JobsBiz.UpdateJobStatusAsync(job.GId, JobStatus.Completed, ct);
+ await JobsBiz.LogJobAsync(job.GId, $"Bulk job {job.SubType} processed {ProcessedObjectCount} of {idList.Count}");
+ await JobsBiz.UpdateJobStatusAsync(job.GId, JobStatus.Completed);
}
diff --git a/server/AyaNova/generator/CoreJobMetricsSnapshot.cs b/server/AyaNova/generator/CoreJobMetricsSnapshot.cs
index a5aa8fa9..5154e988 100644
--- a/server/AyaNova/generator/CoreJobMetricsSnapshot.cs
+++ b/server/AyaNova/generator/CoreJobMetricsSnapshot.cs
@@ -27,17 +27,18 @@ namespace AyaNova.Biz
#if(DEBUG)
private static TimeSpan tsOneMinute = new TimeSpan(0, 0, 10);
- #else
+#else
private static TimeSpan tsOneMinute = new TimeSpan(0, 1, 0);
- #endif
+#endif
private static TimeSpan tsOneHour = new TimeSpan(1, 0, 0);
private static TimeSpan ts24Hours = new TimeSpan(24, 0, 0);
////////////////////////////////////////////////////////////////////////////////////////////////
// DoAsync
//
- public static async Task DoJobAsync(AyContext ct)
+ public static void DoJob()
{
+ #region worksheet
//https://github.com/sebastienros/memoryleak/blob/master/src/MemoryLeak/MemoryLeak/Controllers/DiagnosticsController.cs
// //DATA TYPES .net to postgres map
//http://www.npgsql.org/doc/types/basic.html
@@ -153,6 +154,7 @@ from generate_series(1, 525600) s(i)
,...);
*/
+ #endregion worksheet
//Nothing is gathered less than one minute frequency
if (!DateUtil.IsAfterDuration(_lastSnapshot, tsOneMinute))
@@ -196,12 +198,15 @@ from generate_series(1, 525600) s(i)
var Gen2 = GC.CollectionCount(2);//integer
var CPU = _cpu;// double precision
-
- //write to db
- MetricMM mm = new MetricMM(Allocated, WorkingSet, PrivateBytes, Gen0, Gen1, Gen2, CPU);
- await ct.MetricMM.AddAsync(mm);
- await ct.SaveChangesAsync();
-
+ System.Diagnostics.Debug.WriteLine($"MM Snapshot, cpu: {CPU}");
+ using (AyContext ct = ServiceProviderProvider.DBContext)
+ {
+ //write to db
+ MetricMM mm = new MetricMM(Allocated, WorkingSet, PrivateBytes, Gen0, Gen1, Gen2, CPU);
+ ct.MetricMM.Add(mm);
+ ct.SaveChanges();
+ System.Diagnostics.Debug.WriteLine("MM SAVED");
+ }
/////////////////////////////////////////////
@@ -270,17 +275,17 @@ from generate_series(1, 525600) s(i)
//
DateTime ClearDate = DateTime.UtcNow - tsDataRetention;
- await ct.Database.ExecuteSqlInterpolatedAsync($"delete from ametricmm where t > {ClearDate}");
+ using (AyContext ct = ServiceProviderProvider.DBContext)
+ {
+ ct.Database.ExecuteSqlInterpolated($"delete from ametricmm where t < {ClearDate}");
+ }
}
_lastSnapshot = now;
- {
- DateTime ClearDate = DateTime.UtcNow - tsDataRetention;
- await ct.Database.ExecuteSqlInterpolatedAsync($"delete from ametricmm where t > {ClearDate}");
- }
+
}
diff --git a/server/AyaNova/generator/CoreJobSweeper.cs b/server/AyaNova/generator/CoreJobSweeper.cs
index 7be0d695..cce42dba 100644
--- a/server/AyaNova/generator/CoreJobSweeper.cs
+++ b/server/AyaNova/generator/CoreJobSweeper.cs
@@ -4,7 +4,7 @@ using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using AyaNova.Models;
-
+using System.Threading;
namespace AyaNova.Biz
{
@@ -28,44 +28,49 @@ namespace AyaNova.Biz
////////////////////////////////////////////////////////////////////////////////////////////////
// DoSweep
//
- public static async Task DoSweepAsync(AyContext ct)
+ public static async Task DoSweepAsync(CancellationToken ctoken)
{
-
+ ctoken.ThrowIfCancellationRequested();
//This will get triggered roughly every minute, but we don't want to sweep that frequently
if (DateTime.UtcNow - lastSweep < SWEEP_EVERY_INTERVAL)
return;
log.LogTrace("Sweep starting");
+ using (AyContext ct = AyaNova.Util.ServiceProviderProvider.DBContext)
+ {
- //SWEEP SUCCESSFUL JOBS
- //calculate cutoff to delete
- DateTime dtDeleteCutoff = DateTime.UtcNow - SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
- await sweepAsync(ct, dtDeleteCutoff, JobStatus.Completed);
+ //SWEEP SUCCESSFUL JOBS
+ //calculate cutoff to delete
+ DateTime dtDeleteCutoff = DateTime.UtcNow - SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
+ await sweepAsync(ct, dtDeleteCutoff, JobStatus.Completed, ctoken);
- //SWEEP FAILED JOBS
- //calculate cutoff to delete
- dtDeleteCutoff = DateTime.UtcNow - FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
- await sweepAsync(ct, dtDeleteCutoff, JobStatus.Failed);
-
- //KILL STUCK JOBS
- //calculate cutoff to delete
- DateTime dtRunningDeadline = DateTime.UtcNow - RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN;
- await killStuckJobsAsync(ct, dtRunningDeadline);
-
- //SWEEP INTERNAL JOB LOG
- //calculate cutoff to delete
- dtDeleteCutoff = DateTime.UtcNow - INTERNAL_JOBS_LOGS_DELETE_AFTER_THIS_TIMESPAN;
- await SweepInternalJobsLogsAsync(ct, dtDeleteCutoff);
+ ctoken.ThrowIfCancellationRequested();
+ //SWEEP FAILED JOBS
+ //calculate cutoff to delete
+ dtDeleteCutoff = DateTime.UtcNow - FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
+ await sweepAsync(ct, dtDeleteCutoff, JobStatus.Failed, ctoken);
+ ctoken.ThrowIfCancellationRequested();
+ //KILL STUCK JOBS
+ //calculate cutoff to delete
+ DateTime dtRunningDeadline = DateTime.UtcNow - RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN;
+ await killStuckJobsAsync(ct, dtRunningDeadline);
+ ctoken.ThrowIfCancellationRequested();
+ //SWEEP INTERNAL JOB LOG
+ //calculate cutoff to delete
+ dtDeleteCutoff = DateTime.UtcNow - INTERNAL_JOBS_LOGS_DELETE_AFTER_THIS_TIMESPAN;
+ await SweepInternalJobsLogsAsync(ct, dtDeleteCutoff);
+ ctoken.ThrowIfCancellationRequested();
+ }
lastSweep = DateTime.UtcNow;
}
- private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus)
+ private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus, CancellationToken ctoken)//AyContext ct,
{
-
+ // AyContext ct = ServiceProviderProvider.DBContext;
//Get the deleteable succeeded jobs list
var jobs = await ct.OpsJob
.AsNoTracking()
@@ -79,11 +84,12 @@ namespace AyaNova.Biz
{
try
{
- await JobsBiz.DeleteJobAndLogAsync(j.GId, ct);
+ ctoken.ThrowIfCancellationRequested();
+ await JobsBiz.RemoveJobAndLogsAsync(j.GId);
}
catch (Exception ex)
{
- log.LogError(ex, "sweepAsync exception calling JobsBiz.DeleteJobAndLogAsync");
+ log.LogError(ex, "sweepAsync exception calling JobsBiz.RemoveJobAndLogsAsync");
//for now just throw it but this needs to be removed when logging added and better handling
throw (ex);
}
@@ -93,12 +99,13 @@ namespace AyaNova.Biz
///
/// Kill jobs that have been stuck in "running" state for too long
- ///
- ///
+ ///
///
///
private static async Task killStuckJobsAsync(AyContext ct, DateTime dtRunningDeadline)
{
+
+ // AyContext ct = ServiceProviderProvider.DBContext;
//Get the deleteable succeeded jobs list
var jobs = await ct.OpsJob
.AsNoTracking()
@@ -120,7 +127,7 @@ namespace AyaNova.Biz
private static async Task SweepInternalJobsLogsAsync(AyContext ct, DateTime dtDeleteCutoff)
{
-
+ // AyContext ct = ServiceProviderProvider.DBContext;
//Get the deleteable list (this is for reporting, could easily just do it in one go)
var logs = await ct.OpsJobLog
.AsNoTracking()
diff --git a/server/AyaNova/generator/Generate.cs b/server/AyaNova/generator/Generate.cs
index 34811b07..55fc03a1 100644
--- a/server/AyaNova/generator/Generate.cs
+++ b/server/AyaNova/generator/Generate.cs
@@ -6,6 +6,7 @@ using Microsoft.Extensions.DependencyInjection;
using AyaNova.Models;
using AyaNova.Api.ControllerHelpers;
using AyaNova.Biz;
+using AyaNova.Util;
namespace AyaNova.Generator
{
@@ -27,6 +28,7 @@ namespace AyaNova.Generator
// private readonly ApiServerState serverState;
private readonly IServiceProvider provider;
+ private const int MAXIMUM_TIME_ALLOWED_FOR_PROCESSING_ALL_JOBS = 1 * 60 * 1000;//1 minutes TEST TEST TEST #####
#if(DEBUG)
private const int GENERATE_SECONDS = 5;
@@ -53,7 +55,11 @@ namespace AyaNova.Generator
-
+ /*
+ todo: improve this
+ it should timeout: https://stackoverflow.com/questions/23476576/cancellationtoken-timeout-vs-task-delay-and-timeout
+ it should never stop running no matter what unless teh server shuts down
+ */
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
@@ -73,32 +79,37 @@ namespace AyaNova.Generator
{
log.LogDebug($"GeneratorService task doing background work.");
- using (IServiceScope scope = provider.CreateScope())
+ ApiServerState serverState = ServiceProviderProvider.ServerState;
+ //=================================================================
+ try
{
- AyContext ct = scope.ServiceProvider.GetRequiredService();
- ApiServerState serverState = scope.ServiceProvider.GetRequiredService();
-
-
-
- //=================================================================
- try
+ if (!serverState.IsOpen)
{
- if (!serverState.IsOpen)
- {
- log.LogDebug($"GeneratorService: ServerState is closed returning without processing jobs, will try again next iteration");
- }
- else
- {
- await JobsBiz.ProcessJobsAsync(ct, serverState);
- }
+ log.LogDebug($"GeneratorService: ServerState is closed returning without processing jobs, will try again next iteration");
}
- catch (Exception ex)
+ else
{
- log.LogError(ex, "Generate::ProcessJobs result in exception error ");
+ System.Diagnostics.Debug.WriteLine($"### GENERATE calling JobsBiz.ProcessJobs");
+ //Before anything capture metrics
+ // await CoreJobMetricsSnapshot.DoJobAsync(ct);
+ //Task.Run(() => CoreJobMetricsSnapshot.DoJob());
+ //deliberately calling this non-async
+ //it needs to be fast and efficient
+ CoreJobMetricsSnapshot.DoJob();
+ //TODO: this should be big timeout and then inside the process jobs each job has it's own timeout
+ await TaskUtil.WithTimeoutAfterStart(ctoken => JobsBiz.ProcessJobsAsync(ctoken), TimeSpan.FromMilliseconds(MAXIMUM_TIME_ALLOWED_FOR_PROCESSING_ALL_JOBS));
+ // await JobsBiz.ProcessJobsAsync(ct, serverState);
+ System.Diagnostics.Debug.WriteLine($"### GENERATE BACK FROM calling JobsBiz.ProcessJobs");
}
- //=================================================================
}
+ catch (Exception ex)
+ {
+ log.LogError(ex, "Generate::ProcessJobs result in exception error ");
+
+ }
+ //=================================================================
+
}
await Task.Delay((GENERATE_SECONDS * 1000), stoppingToken);
justStarted = false;
diff --git a/server/AyaNova/util/ServiceProviderProvider.cs b/server/AyaNova/util/ServiceProviderProvider.cs
index 714dac0b..607a5f97 100644
--- a/server/AyaNova/util/ServiceProviderProvider.cs
+++ b/server/AyaNova/util/ServiceProviderProvider.cs
@@ -58,6 +58,14 @@ namespace AyaNova.Util
}
}
+ internal static AyaNova.Api.ControllerHelpers.ApiServerState ServerState
+ {
+ get
+ {
+ return Scope.ServiceProvider.GetRequiredService();
+ }
+ }
+
// https://docs.microsoft.com/en-us/aspnet/core/fundamentals/http-requests?view=aspnetcore-3.1
internal static System.Net.Http.IHttpClientFactory HttpClientFactory
{