diff --git a/server/AyaNova/Controllers/BackupController.cs b/server/AyaNova/Controllers/BackupController.cs index 07f6dc29..f2a0fdac 100644 --- a/server/AyaNova/Controllers/BackupController.cs +++ b/server/AyaNova/Controllers/BackupController.cs @@ -68,7 +68,7 @@ namespace AyaNova.Api.Controllers j.JobType = JobType.Backup; j.SubType = JobSubType.NotSet; j.Exclusive = true; - await JobsBiz.AddJobAsync(j, ct); + await JobsBiz.AddJobAsync(j); await EventLogProcessor.LogEventToDatabaseAsync(new Event(UserIdFromContext.Id(HttpContext.Items), 0, AyaType.ServerJob, AyaEvent.Created, JobName), ct); return Accepted(new { JobId = j.GId }); } diff --git a/server/AyaNova/biz/BizObjectFactory.cs b/server/AyaNova/biz/BizObjectFactory.cs index 8f752f86..50b2cb92 100644 --- a/server/AyaNova/biz/BizObjectFactory.cs +++ b/server/AyaNova/biz/BizObjectFactory.cs @@ -20,8 +20,9 @@ namespace AyaNova.Biz //Returns the biz object class that corresponds to the type presented //Used by SEARCH and objects with JOBS - internal static BizObject GetBizObject(AyaType ayaType, AyContext dbcontext, long userId = 1, AuthorizationRoles roles = AuthorizationRoles.All) + internal static BizObject GetBizObject(AyaType ayaType, long userId = 1, AuthorizationRoles roles = AuthorizationRoles.All) { + var dbcontext = ServiceProviderProvider.DBContext; switch (ayaType) { //CoreBizObject add here diff --git a/server/AyaNova/biz/JobsBiz.cs b/server/AyaNova/biz/JobsBiz.cs index 8cfbc6b0..f592c340 100644 --- a/server/AyaNova/biz/JobsBiz.cs +++ b/server/AyaNova/biz/JobsBiz.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using System.Threading.Tasks; +using System.Threading; using System.Collections.Generic; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; @@ -22,10 +23,11 @@ namespace AyaNova.Biz /// Get a non tracked list of jobs for an object /// /// - /// + /// /// - internal static async Task> GetJobsForObjectAsync(AyaTypeId ayObj, AyContext ct) + internal static async Task> GetJobsForObjectAsync(AyaTypeId ayObj) { + var ct = ServiceProviderProvider.DBContext; return await ct.OpsJob .AsNoTracking() .Where(z => z.ObjectId == ayObj.ObjectId && z.ObjectType == ayObj.ObjectType) @@ -39,9 +41,9 @@ namespace AyaNova.Biz /// Get a non tracked list of jobs that are ready to process and exclusive only /// /// - internal static async Task> GetReadyJobsExclusiveOnlyAsync(AyContext ct) + internal static async Task> GetReadyJobsExclusiveOnlyAsync() { - return await GetReadyJobsAsync(true, ct); + return await GetReadyJobsAsync(true); } @@ -49,9 +51,9 @@ namespace AyaNova.Biz /// Get a non tracked list of jobs that are ready to process and exclusive only /// /// - internal static async Task> GetReadyJobsNotExlusiveOnlyAsync(AyContext ct) + internal static async Task> GetReadyJobsNotExlusiveOnlyAsync() { - return await GetReadyJobsAsync(false, ct); + return await GetReadyJobsAsync(false); } @@ -59,8 +61,9 @@ namespace AyaNova.Biz /// Get a non tracked list of jobs filtered by exclusivity /// /// - private static async Task> GetReadyJobsAsync(bool exclusiveOnly, AyContext ct) + private static async Task> GetReadyJobsAsync(bool exclusiveOnly) { + var ct = ServiceProviderProvider.DBContext; var ret = await ct.OpsJob .AsNoTracking() .Where(z => z.StartAfter < System.DateTime.UtcNow && z.Exclusive == exclusiveOnly && z.JobStatus == JobStatus.Sleeping) @@ -76,8 +79,9 @@ namespace AyaNova.Biz /// could be running or sleeping /// /// - internal static async Task> GetAllSleepingOrRunningJobsAsync(AyContext ct) + internal static async Task> GetAllSleepingOrRunningJobsAsync() { + var ct = ServiceProviderProvider.DBContext; var ret = await ct.OpsJob .AsNoTracking() .Where(z => z.JobStatus == JobStatus.Sleeping || z.JobStatus == JobStatus.Running) @@ -92,8 +96,9 @@ namespace AyaNova.Biz /// Get a non tracked list of all jobs for a JobType /// /// - internal static async Task> GetAllJobsForJobTypeAsync(AyContext ct, JobType jobType) + internal static async Task> GetAllJobsForJobTypeAsync(JobType jobType) { + var ct = ServiceProviderProvider.DBContext; var ret = await ct.OpsJob .AsNoTracking() .Where(z => z.JobType == jobType) @@ -108,8 +113,9 @@ namespace AyaNova.Biz /// Get a non tracked list of all jobs that are status running but have no last activity for XX HOURS /// /// - internal static async Task> GetPotentiallyDeadRunningJobsAsync(AyContext ct) + internal static async Task> GetPotentiallyDeadRunningJobsAsync() { + var ct = ServiceProviderProvider.DBContext; var ret = await ct.OpsJob .AsNoTracking() .Where(z => z.JobStatus == JobStatus.Sleeping || z.JobStatus == JobStatus.Running) @@ -124,8 +130,9 @@ namespace AyaNova.Biz /// Get a count of all jobs for a JobStatus /// /// - internal static async Task GetCountForJobStatusAsync(AyContext ct, JobStatus jobStatus) + internal static async Task GetCountForJobStatusAsync(JobStatus jobStatus) { + var ct = ServiceProviderProvider.DBContext; var ret = await ct.OpsJob .Where(z => z.JobStatus == jobStatus) .LongCountAsync(); @@ -138,10 +145,11 @@ namespace AyaNova.Biz /// Add a new job to the database /// /// - /// + /// - internal static async Task AddJobAsync(OpsJob newJob, AyContext ct) + internal static async Task AddJobAsync(OpsJob newJob) { + var ct = ServiceProviderProvider.DBContext; await ct.OpsJob.AddAsync(newJob); await ct.SaveChangesAsync(); return newJob; @@ -151,110 +159,57 @@ namespace AyaNova.Biz /// Remove any jobs or logs for the object in question /// /// - /// + /// internal static async Task DeleteJobsForObjectAsync(AyaTypeId ayObj, AyContext ct) { //Get a list of all jobid's for the object passed in - List jobsForObject = await GetJobsForObjectAsync(ayObj, ct); - - //short circuit + List jobsForObject = await GetJobsForObjectAsync(ayObj); if (jobsForObject.Count == 0) return; - - using (var transaction = await ct.Database.BeginTransactionAsync()) + foreach (OpsJob jobToBeDeleted in jobsForObject) { - try - { - foreach (OpsJob jobToBeDeleted in jobsForObject) - { - await RemoveJobAndLogsAsync(ct, jobToBeDeleted.GId); - } - - // Commit transaction if all commands succeed, transaction will auto-rollback - // when disposed if either commands fails - await transaction.CommitAsync(); - } - catch (Exception ex) - { - throw ex; - } + await RemoveJobAndLogsAsync(jobToBeDeleted.GId); } } - - - /// - /// Remove job and logs for that job - /// - /// - /// - internal static async Task DeleteJobAndLogAsync(Guid jobId, AyContext ct) - { - using (var transaction = await ct.Database.BeginTransactionAsync()) - { - try - { - await RemoveJobAndLogsAsync(ct, jobId); - // Commit transaction if all commands succeed, transaction will auto-rollback - // when disposed if either commands fails - await transaction.CommitAsync(); - } - catch (Exception ex) - { - throw ex; - } - } - } - - - /// /// REmove the job and it's logs - /// - /// + /// /// - private static async Task RemoveJobAndLogsAsync(AyContext ct, Guid jobIdToBeDeleted) + internal static async Task RemoveJobAndLogsAsync(Guid jobIdToBeDeleted) { - // //delete logs - // await ct.Database.ExecuteSqlCommandAsync("delete from aopsjoblog where jobid = {0}", new object[] { jobIdToBeDeleted }); - - // //delete the job - // await ct.Database.ExecuteSqlCommandAsync("delete from aopsjob where gid = {0}", new object[] { jobIdToBeDeleted }); - - //delete logs - await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjoblog where jobid = {jobIdToBeDeleted}"); - - //delete the job - await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjob where gid = {jobIdToBeDeleted}"); + var ct = ServiceProviderProvider.DBContext; + using (var transaction = await ct.Database.BeginTransactionAsync()) + { + try + { + //delete logs + await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjoblog where jobid = {jobIdToBeDeleted}"); + //delete the job + await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjob where gid = {jobIdToBeDeleted}"); + // Commit transaction if all commands succeed, transaction will auto-rollback + // when disposed if either commands fails + await transaction.CommitAsync(); + } + catch (Exception ex) + { + throw ex; + } + } } - - - /// /// Make a log entry for a job - /// (no context version) /// /// (NOTE: Guid.empty indicates internal job) /// + /// internal static async Task LogJobAsync(Guid jobId, string statusText) { - return await LogJobAsync(jobId, statusText, null); - } - - /// - /// Make a log entry for a job - /// - /// (NOTE: Guid.empty indicates internal job) - /// - /// - internal static async Task LogJobAsync(Guid jobId, string statusText, AyContext ct) - { - if (ct == null) - ct = ServiceProviderProvider.DBContext; + var ct = ServiceProviderProvider.DBContext; if (string.IsNullOrWhiteSpace(statusText)) statusText = "No status provided"; OpsJobLog newObj = new OpsJobLog(); @@ -270,10 +225,10 @@ namespace AyaNova.Biz /// Update the status of a job /// /// - /// - /// - internal static async Task UpdateJobStatusAsync(Guid jobId, JobStatus newStatus, AyContext ct) + /// + internal static async Task UpdateJobStatusAsync(Guid jobId, JobStatus newStatus) { + var ct = ServiceProviderProvider.DBContext; var oFromDb = await ct.OpsJob.SingleOrDefaultAsync(z => z.GId == jobId); if (oFromDb == null) return null; oFromDb.JobStatus = newStatus; @@ -289,37 +244,70 @@ namespace AyaNova.Biz /// Process all jobs (stock jobs and those found in operations table) /// /// - internal static async Task ProcessJobsAsync(AyContext ct, AyaNova.Api.ControllerHelpers.ApiServerState serverState) + internal static async Task ProcessJobsAsync(CancellationToken ctoken) { - if (ActivelyProcessing) { + System.Diagnostics.Debug.WriteLine("ProcessJobs called but actively processing other jobs so returning"); log.LogTrace("ProcessJobs called but actively processing other jobs so returning"); return; } ActivelyProcessing = true; try { - + ctoken.ThrowIfCancellationRequested(); + + //Sweep jobs table + System.Diagnostics.Debug.WriteLine($"JobsBiz processing sweeper"); + await CoreJobSweeper.DoSweepAsync(ctoken);//run exclusively + //BIZOBJECT DYNAMIC JOBS //get a list of exclusive jobs that are due to happen //Call into each item in turn - List exclusiveJobs = await GetReadyJobsExclusiveOnlyAsync(ct); + List exclusiveJobs = await GetReadyJobsExclusiveOnlyAsync(); foreach (OpsJob j in exclusiveJobs) { try { - await ProcessJobAsync(j, ct); + ctoken.ThrowIfCancellationRequested(); + System.Diagnostics.Debug.WriteLine($"JobsBiz processing exclusive biz job {j.Name}"); + await ProcessJobAsync(j); } catch (Exception ex) { log.LogError(ex, $"ProcessJobs::Exclusive -> job {j.Name} failed with exception"); - await LogJobAsync(j.GId, "Job failed with errors:", ct); - await LogJobAsync(j.GId, ExceptionUtil.ExtractAllExceptionMessages(ex), ct); - await UpdateJobStatusAsync(j.GId, JobStatus.Failed, ct); + await LogJobAsync(j.GId, "Job failed with errors:"); + await LogJobAsync(j.GId, ExceptionUtil.ExtractAllExceptionMessages(ex)); + await UpdateJobStatusAsync(j.GId, JobStatus.Failed); } } - //Get a list of non-exlusive jobs that are due + + ctoken.ThrowIfCancellationRequested(); + System.Diagnostics.Debug.WriteLine($"JobsBiz processing exclusive license check"); + //License check + long CurrentActiveCount = await UserBiz.ActiveCountAsync(); + long LicensedUserCount = AyaNova.Core.License.ActiveKey.ActiveNumber; + // log.LogInformation("JobsBiz::Checking license active count"); + if (CurrentActiveCount > LicensedUserCount) + { + var msg = $"E1020 - Active count exceeded capacity"; + ServiceProviderProvider.ServerState.SetSystemLock(msg); + log.LogCritical(msg); + return; + } + ctoken.ThrowIfCancellationRequested(); + //backup + System.Diagnostics.Debug.WriteLine($"JobsBiz processing backup"); + await CoreJobBackup.DoWorkAsync();//sb exclusive + System.Diagnostics.Debug.WriteLine($"JobsBiz processing metrics snapshotter"); + + + /////////////////////////////////////// + //NON-EXCLUSIVE JOBS + // + + + //LOOKAT: Parallelize / background this block //http://www.dotnetcurry.com/dotnet/1360/concurrent-programming-dotnet-core @@ -329,42 +317,31 @@ namespace AyaNova.Biz //This area may turn out to need a re-write in future, but I think it might only involve this block and ProcessJobAsync //the actual individual objects that are responsible for jobs will likely not need a signature rewrite or anything (I hope) //For now I'z hoping that no job will be so slow that it can hold up all the other jobs indefinitely. - - List sharedJobs = await GetReadyJobsNotExlusiveOnlyAsync(ct); + ctoken.ThrowIfCancellationRequested(); + List sharedJobs = await GetReadyJobsNotExlusiveOnlyAsync(); foreach (OpsJob j in sharedJobs) { try { - await ProcessJobAsync(j, ct); + ctoken.ThrowIfCancellationRequested(); + System.Diagnostics.Debug.WriteLine($"JobsBiz processing NON-exclusive biz job {j.Name}"); + // Task.Run(() => FireAway()); + Task.Run(() => ProcessJobAsync(j), ctoken); + // await ProcessJobAsync(j, ct); } catch (Exception ex) { log.LogError(ex, $"ProcessJobs::Shared -> job {j.Name} failed with exception"); - await LogJobAsync(j.GId, "Job failed with errors:", ct); - await LogJobAsync(j.GId, ExceptionUtil.ExtractAllExceptionMessages(ex), ct); - await UpdateJobStatusAsync(j.GId, JobStatus.Failed, ct); + await LogJobAsync(j.GId, "Job failed with errors:"); + await LogJobAsync(j.GId, ExceptionUtil.ExtractAllExceptionMessages(ex)); + await UpdateJobStatusAsync(j.GId, JobStatus.Failed); } } - //STOCK JOBS - //Sweep jobs table - await CoreJobSweeper.DoSweepAsync(ct); - //License check - long CurrentActiveCount = await UserBiz.ActiveCountAsync(); - long LicensedUserCount = AyaNova.Core.License.ActiveKey.ActiveNumber; - // log.LogInformation("JobsBiz::Checking license active count"); - if (CurrentActiveCount > LicensedUserCount) - { - var msg = $"E1020 - Active count exceeded capacity"; - serverState.SetSystemLock(msg); - log.LogCritical(msg); - return; - } + //STOCK JOBS + - //backup - await CoreJobBackup.DoWorkAsync(ct); - await CoreJobMetricsSnapshot.DoJobAsync(ct); //Notifications @@ -377,6 +354,7 @@ namespace AyaNova.Biz finally { ActivelyProcessing = false; + System.Diagnostics.Debug.WriteLine($"JobsBiz in Finally - completed run"); } } @@ -384,15 +362,14 @@ namespace AyaNova.Biz /// /// Process a job by calling into it's biz object /// - /// - /// + /// /// - internal static async Task ProcessJobAsync(OpsJob job, AyContext ct) + internal static async Task ProcessJobAsync(OpsJob job) { var JobDescription = $"{job.Name} {job.JobType.ToString()}"; if (job.SubType != JobSubType.NotSet) JobDescription += $":{job.SubType}"; - await LogJobAsync(job.GId, $"Process job \"{JobDescription}\"", ct); + await LogJobAsync(job.GId, $"Process job \"{JobDescription}\""); log.LogDebug($"ProcessJobAsync -> Processing job {JobDescription}"); IJobObject o = null; @@ -400,19 +377,19 @@ namespace AyaNova.Biz { case JobType.Backup: //This is called when on demand only, normal backups are processed above with normal system jobs - await CoreJobBackup.DoWorkAsync(ct, true); - await UpdateJobStatusAsync(job.GId, JobStatus.Completed, ct); + await CoreJobBackup.DoWorkAsync(true); + await UpdateJobStatusAsync(job.GId, JobStatus.Completed); break; case JobType.TestJob: - o = (IJobObject)BizObjectFactory.GetBizObject(AyaType.ServerJob, ct); + o = (IJobObject)BizObjectFactory.GetBizObject(AyaType.ServerJob); break; case JobType.SeedTestData: - o = (IJobObject)BizObjectFactory.GetBizObject(AyaType.TrialSeeder, ct); + o = (IJobObject)BizObjectFactory.GetBizObject(AyaType.TrialSeeder); break; case JobType.BulkCoreBizObjectOperation: //bulk op, hand off to biz object to deal with //note, convention is that there is an idList in job.jobinfo json if preselected else it's all objects of type - o = (IJobObject)BizObjectFactory.GetBizObject(job.ObjectType, ct); + o = (IJobObject)BizObjectFactory.GetBizObject(job.ObjectType); break; default: throw new System.NotSupportedException($"ProcessJobAsync type {job.JobType.ToString()} is not supported"); diff --git a/server/AyaNova/biz/Search.cs b/server/AyaNova/biz/Search.cs index 5a64ee32..630e2be7 100644 --- a/server/AyaNova/biz/Search.cs +++ b/server/AyaNova/biz/Search.cs @@ -246,8 +246,7 @@ namespace AyaNova.Biz #endregion dosearch #region Get info (excerpt) - public static async Task GetInfoAsync(AyContext ct, long translationId, - AuthorizationRoles currentUserRoles, long userId, string phrase, int max, AyaType ayaType, long id) + public static async Task GetInfoAsync(long translationId, AuthorizationRoles currentUserRoles, long userId, string phrase, int max, AyaType ayaType, long id) { //escape literal percentage signs first just in case they are searching for 50% off or something //https://www.postgresql.org/docs/current/functions-matching.html#FUNCTIONS-LIKE @@ -262,7 +261,7 @@ namespace AyaNova.Biz PhraseItems.ToArray(); //get text - ISearchAbleObject o = (ISearchAbleObject)BizObjectFactory.GetBizObject(ayaType, ct, userId, currentUserRoles); + ISearchAbleObject o = (ISearchAbleObject)BizObjectFactory.GetBizObject(ayaType, userId, currentUserRoles); //get extract var searchParams = await o.GetSearchResultSummary(id); diff --git a/server/AyaNova/biz/WorkOrderBiz.cs b/server/AyaNova/biz/WorkOrderBiz.cs index c4719ddc..54bd63d2 100644 --- a/server/AyaNova/biz/WorkOrderBiz.cs +++ b/server/AyaNova/biz/WorkOrderBiz.cs @@ -2441,8 +2441,8 @@ namespace AyaNova.Biz private async Task ProcessBulkJobAsync(OpsJob job) { - await JobsBiz.UpdateJobStatusAsync(job.GId, JobStatus.Running, ct); - await JobsBiz.LogJobAsync(job.GId, $"Bulk job {job.SubType} started...", ct); + await JobsBiz.UpdateJobStatusAsync(job.GId, JobStatus.Running); + await JobsBiz.LogJobAsync(job.GId, $"Bulk job {job.SubType} started..."); List idList = new List(); long ProcessedObjectCount = 0; JObject jobData = JObject.Parse(job.JobInfo); @@ -2475,7 +2475,7 @@ namespace AyaNova.Biz { o = await PutWorkOrderGraphItem(job.ObjectType, o); if (o == null) - await JobsBiz.LogJobAsync(job.GId, $"Error processing item {id}: {GetErrorsAsString()}", ct); + await JobsBiz.LogJobAsync(job.GId, $"Error processing item {id}: {GetErrorsAsString()}"); else ProcessedObjectCount++; } @@ -2484,12 +2484,12 @@ namespace AyaNova.Biz } catch (Exception ex) { - await JobsBiz.LogJobAsync(job.GId, $"Error processing item {id}", ct); - await JobsBiz.LogJobAsync(job.GId, ExceptionUtil.ExtractAllExceptionMessages(ex), ct); + await JobsBiz.LogJobAsync(job.GId, $"Error processing item {id}"); + await JobsBiz.LogJobAsync(job.GId, ExceptionUtil.ExtractAllExceptionMessages(ex)); } } - await JobsBiz.LogJobAsync(job.GId, $"Bulk job {job.SubType} processed {ProcessedObjectCount} of {idList.Count}", ct); - await JobsBiz.UpdateJobStatusAsync(job.GId, JobStatus.Completed, ct); + await JobsBiz.LogJobAsync(job.GId, $"Bulk job {job.SubType} processed {ProcessedObjectCount} of {idList.Count}"); + await JobsBiz.UpdateJobStatusAsync(job.GId, JobStatus.Completed); } diff --git a/server/AyaNova/generator/CoreJobMetricsSnapshot.cs b/server/AyaNova/generator/CoreJobMetricsSnapshot.cs index a5aa8fa9..5154e988 100644 --- a/server/AyaNova/generator/CoreJobMetricsSnapshot.cs +++ b/server/AyaNova/generator/CoreJobMetricsSnapshot.cs @@ -27,17 +27,18 @@ namespace AyaNova.Biz #if(DEBUG) private static TimeSpan tsOneMinute = new TimeSpan(0, 0, 10); - #else +#else private static TimeSpan tsOneMinute = new TimeSpan(0, 1, 0); - #endif +#endif private static TimeSpan tsOneHour = new TimeSpan(1, 0, 0); private static TimeSpan ts24Hours = new TimeSpan(24, 0, 0); //////////////////////////////////////////////////////////////////////////////////////////////// // DoAsync // - public static async Task DoJobAsync(AyContext ct) + public static void DoJob() { + #region worksheet //https://github.com/sebastienros/memoryleak/blob/master/src/MemoryLeak/MemoryLeak/Controllers/DiagnosticsController.cs // //DATA TYPES .net to postgres map //http://www.npgsql.org/doc/types/basic.html @@ -153,6 +154,7 @@ from generate_series(1, 525600) s(i) ,...); */ + #endregion worksheet //Nothing is gathered less than one minute frequency if (!DateUtil.IsAfterDuration(_lastSnapshot, tsOneMinute)) @@ -196,12 +198,15 @@ from generate_series(1, 525600) s(i) var Gen2 = GC.CollectionCount(2);//integer var CPU = _cpu;// double precision - - //write to db - MetricMM mm = new MetricMM(Allocated, WorkingSet, PrivateBytes, Gen0, Gen1, Gen2, CPU); - await ct.MetricMM.AddAsync(mm); - await ct.SaveChangesAsync(); - + System.Diagnostics.Debug.WriteLine($"MM Snapshot, cpu: {CPU}"); + using (AyContext ct = ServiceProviderProvider.DBContext) + { + //write to db + MetricMM mm = new MetricMM(Allocated, WorkingSet, PrivateBytes, Gen0, Gen1, Gen2, CPU); + ct.MetricMM.Add(mm); + ct.SaveChanges(); + System.Diagnostics.Debug.WriteLine("MM SAVED"); + } ///////////////////////////////////////////// @@ -270,17 +275,17 @@ from generate_series(1, 525600) s(i) // DateTime ClearDate = DateTime.UtcNow - tsDataRetention; - await ct.Database.ExecuteSqlInterpolatedAsync($"delete from ametricmm where t > {ClearDate}"); + using (AyContext ct = ServiceProviderProvider.DBContext) + { + ct.Database.ExecuteSqlInterpolated($"delete from ametricmm where t < {ClearDate}"); + } } _lastSnapshot = now; - { - DateTime ClearDate = DateTime.UtcNow - tsDataRetention; - await ct.Database.ExecuteSqlInterpolatedAsync($"delete from ametricmm where t > {ClearDate}"); - } + } diff --git a/server/AyaNova/generator/CoreJobSweeper.cs b/server/AyaNova/generator/CoreJobSweeper.cs index 7be0d695..cce42dba 100644 --- a/server/AyaNova/generator/CoreJobSweeper.cs +++ b/server/AyaNova/generator/CoreJobSweeper.cs @@ -4,7 +4,7 @@ using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using AyaNova.Models; - +using System.Threading; namespace AyaNova.Biz { @@ -28,44 +28,49 @@ namespace AyaNova.Biz //////////////////////////////////////////////////////////////////////////////////////////////// // DoSweep // - public static async Task DoSweepAsync(AyContext ct) + public static async Task DoSweepAsync(CancellationToken ctoken) { - + ctoken.ThrowIfCancellationRequested(); //This will get triggered roughly every minute, but we don't want to sweep that frequently if (DateTime.UtcNow - lastSweep < SWEEP_EVERY_INTERVAL) return; log.LogTrace("Sweep starting"); + using (AyContext ct = AyaNova.Util.ServiceProviderProvider.DBContext) + { - //SWEEP SUCCESSFUL JOBS - //calculate cutoff to delete - DateTime dtDeleteCutoff = DateTime.UtcNow - SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN; - await sweepAsync(ct, dtDeleteCutoff, JobStatus.Completed); + //SWEEP SUCCESSFUL JOBS + //calculate cutoff to delete + DateTime dtDeleteCutoff = DateTime.UtcNow - SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN; + await sweepAsync(ct, dtDeleteCutoff, JobStatus.Completed, ctoken); - //SWEEP FAILED JOBS - //calculate cutoff to delete - dtDeleteCutoff = DateTime.UtcNow - FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN; - await sweepAsync(ct, dtDeleteCutoff, JobStatus.Failed); - - //KILL STUCK JOBS - //calculate cutoff to delete - DateTime dtRunningDeadline = DateTime.UtcNow - RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN; - await killStuckJobsAsync(ct, dtRunningDeadline); - - //SWEEP INTERNAL JOB LOG - //calculate cutoff to delete - dtDeleteCutoff = DateTime.UtcNow - INTERNAL_JOBS_LOGS_DELETE_AFTER_THIS_TIMESPAN; - await SweepInternalJobsLogsAsync(ct, dtDeleteCutoff); + ctoken.ThrowIfCancellationRequested(); + //SWEEP FAILED JOBS + //calculate cutoff to delete + dtDeleteCutoff = DateTime.UtcNow - FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN; + await sweepAsync(ct, dtDeleteCutoff, JobStatus.Failed, ctoken); + ctoken.ThrowIfCancellationRequested(); + //KILL STUCK JOBS + //calculate cutoff to delete + DateTime dtRunningDeadline = DateTime.UtcNow - RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN; + await killStuckJobsAsync(ct, dtRunningDeadline); + ctoken.ThrowIfCancellationRequested(); + //SWEEP INTERNAL JOB LOG + //calculate cutoff to delete + dtDeleteCutoff = DateTime.UtcNow - INTERNAL_JOBS_LOGS_DELETE_AFTER_THIS_TIMESPAN; + await SweepInternalJobsLogsAsync(ct, dtDeleteCutoff); + ctoken.ThrowIfCancellationRequested(); + } lastSweep = DateTime.UtcNow; } - private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus) + private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus, CancellationToken ctoken)//AyContext ct, { - + // AyContext ct = ServiceProviderProvider.DBContext; //Get the deleteable succeeded jobs list var jobs = await ct.OpsJob .AsNoTracking() @@ -79,11 +84,12 @@ namespace AyaNova.Biz { try { - await JobsBiz.DeleteJobAndLogAsync(j.GId, ct); + ctoken.ThrowIfCancellationRequested(); + await JobsBiz.RemoveJobAndLogsAsync(j.GId); } catch (Exception ex) { - log.LogError(ex, "sweepAsync exception calling JobsBiz.DeleteJobAndLogAsync"); + log.LogError(ex, "sweepAsync exception calling JobsBiz.RemoveJobAndLogsAsync"); //for now just throw it but this needs to be removed when logging added and better handling throw (ex); } @@ -93,12 +99,13 @@ namespace AyaNova.Biz /// /// Kill jobs that have been stuck in "running" state for too long - /// - /// + /// /// /// private static async Task killStuckJobsAsync(AyContext ct, DateTime dtRunningDeadline) { + + // AyContext ct = ServiceProviderProvider.DBContext; //Get the deleteable succeeded jobs list var jobs = await ct.OpsJob .AsNoTracking() @@ -120,7 +127,7 @@ namespace AyaNova.Biz private static async Task SweepInternalJobsLogsAsync(AyContext ct, DateTime dtDeleteCutoff) { - + // AyContext ct = ServiceProviderProvider.DBContext; //Get the deleteable list (this is for reporting, could easily just do it in one go) var logs = await ct.OpsJobLog .AsNoTracking() diff --git a/server/AyaNova/generator/Generate.cs b/server/AyaNova/generator/Generate.cs index 34811b07..55fc03a1 100644 --- a/server/AyaNova/generator/Generate.cs +++ b/server/AyaNova/generator/Generate.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.DependencyInjection; using AyaNova.Models; using AyaNova.Api.ControllerHelpers; using AyaNova.Biz; +using AyaNova.Util; namespace AyaNova.Generator { @@ -27,6 +28,7 @@ namespace AyaNova.Generator // private readonly ApiServerState serverState; private readonly IServiceProvider provider; + private const int MAXIMUM_TIME_ALLOWED_FOR_PROCESSING_ALL_JOBS = 1 * 60 * 1000;//1 minutes TEST TEST TEST ##### #if(DEBUG) private const int GENERATE_SECONDS = 5; @@ -53,7 +55,11 @@ namespace AyaNova.Generator - + /* + todo: improve this + it should timeout: https://stackoverflow.com/questions/23476576/cancellationtoken-timeout-vs-task-delay-and-timeout + it should never stop running no matter what unless teh server shuts down + */ protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -73,32 +79,37 @@ namespace AyaNova.Generator { log.LogDebug($"GeneratorService task doing background work."); - using (IServiceScope scope = provider.CreateScope()) + ApiServerState serverState = ServiceProviderProvider.ServerState; + //================================================================= + try { - AyContext ct = scope.ServiceProvider.GetRequiredService(); - ApiServerState serverState = scope.ServiceProvider.GetRequiredService(); - - - - //================================================================= - try + if (!serverState.IsOpen) { - if (!serverState.IsOpen) - { - log.LogDebug($"GeneratorService: ServerState is closed returning without processing jobs, will try again next iteration"); - } - else - { - await JobsBiz.ProcessJobsAsync(ct, serverState); - } + log.LogDebug($"GeneratorService: ServerState is closed returning without processing jobs, will try again next iteration"); } - catch (Exception ex) + else { - log.LogError(ex, "Generate::ProcessJobs result in exception error "); + System.Diagnostics.Debug.WriteLine($"### GENERATE calling JobsBiz.ProcessJobs"); + //Before anything capture metrics + // await CoreJobMetricsSnapshot.DoJobAsync(ct); + //Task.Run(() => CoreJobMetricsSnapshot.DoJob()); + //deliberately calling this non-async + //it needs to be fast and efficient + CoreJobMetricsSnapshot.DoJob(); + //TODO: this should be big timeout and then inside the process jobs each job has it's own timeout + await TaskUtil.WithTimeoutAfterStart(ctoken => JobsBiz.ProcessJobsAsync(ctoken), TimeSpan.FromMilliseconds(MAXIMUM_TIME_ALLOWED_FOR_PROCESSING_ALL_JOBS)); + // await JobsBiz.ProcessJobsAsync(ct, serverState); + System.Diagnostics.Debug.WriteLine($"### GENERATE BACK FROM calling JobsBiz.ProcessJobs"); } - //================================================================= } + catch (Exception ex) + { + log.LogError(ex, "Generate::ProcessJobs result in exception error "); + + } + //================================================================= + } await Task.Delay((GENERATE_SECONDS * 1000), stoppingToken); justStarted = false; diff --git a/server/AyaNova/util/ServiceProviderProvider.cs b/server/AyaNova/util/ServiceProviderProvider.cs index 714dac0b..607a5f97 100644 --- a/server/AyaNova/util/ServiceProviderProvider.cs +++ b/server/AyaNova/util/ServiceProviderProvider.cs @@ -58,6 +58,14 @@ namespace AyaNova.Util } } + internal static AyaNova.Api.ControllerHelpers.ApiServerState ServerState + { + get + { + return Scope.ServiceProvider.GetRequiredService(); + } + } + // https://docs.microsoft.com/en-us/aspnet/core/fundamentals/http-requests?view=aspnetcore-3.1 internal static System.Net.Http.IHttpClientFactory HttpClientFactory {