admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

much fuckery

Browse Source
This commit is contained in:
John Cardinal
2020-05-25 20:34:11 +00:00
parent d977728a22
commit e90dbe0f71
9 changed files with 218 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/AyaNova/Controllers/BackupController.cs
View File

@@ -68,7 +68,7 @@ namespace AyaNova.Api.Controllers
j.JobType = JobType.Backup;
j.SubType = JobSubType.NotSet;
j.Exclusive = true;
await JobsBiz.AddJobAsync(j, ct);
await JobsBiz.AddJobAsync(j);
await EventLogProcessor.LogEventToDatabaseAsync(new Event(UserIdFromContext.Id(HttpContext.Items), 0, AyaType.ServerJob, AyaEvent.Created, JobName), ct);
return Accepted(new { JobId = j.GId });
}

3
server/AyaNova/biz/BizObjectFactory.cs
View File

@@ -20,8 +20,9 @@ namespace AyaNova.Biz
//Returns the biz object class that corresponds to the type presented
//Used by SEARCH and objects with JOBS
internal static BizObject GetBizObject(AyaType ayaType, AyContext dbcontext, long userId = 1, AuthorizationRoles roles = AuthorizationRoles.All)
internal static BizObject GetBizObject(AyaType ayaType, long userId = 1, AuthorizationRoles roles = AuthorizationRoles.All)
{
var dbcontext = ServiceProviderProvider.DBContext;
switch (ayaType)
{
//CoreBizObject add here

249
server/AyaNova/biz/JobsBiz.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using System.Threading;
using System.Collections.Generic;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
@@ -22,10 +23,11 @@ namespace AyaNova.Biz
/// Get a non tracked list of jobs for an object
/// </summary>
/// <param name="ayObj"></param>
/// <param name="ct"></param>
///
/// <returns></returns>
internal static async Task<List<OpsJob>> GetJobsForObjectAsync(AyaTypeId ayObj, AyContext ct)
internal static async Task<List<OpsJob>> GetJobsForObjectAsync(AyaTypeId ayObj)
{
var ct = ServiceProviderProvider.DBContext;
return await ct.OpsJob
.AsNoTracking()
.Where(z => z.ObjectId == ayObj.ObjectId && z.ObjectType == ayObj.ObjectType)
@@ -39,9 +41,9 @@ namespace AyaNova.Biz
/// Get a non tracked list of jobs that are ready to process and exclusive only
/// </summary>
/// <returns></returns>
internal static async Task<List<OpsJob>> GetReadyJobsExclusiveOnlyAsync(AyContext ct)
internal static async Task<List<OpsJob>> GetReadyJobsExclusiveOnlyAsync()
{
return await GetReadyJobsAsync(true, ct);
return await GetReadyJobsAsync(true);
}
@@ -49,9 +51,9 @@ namespace AyaNova.Biz
/// Get a non tracked list of jobs that are ready to process and exclusive only
/// </summary>
/// <returns></returns>
internal static async Task<List<OpsJob>> GetReadyJobsNotExlusiveOnlyAsync(AyContext ct)
internal static async Task<List<OpsJob>> GetReadyJobsNotExlusiveOnlyAsync()
{
return await GetReadyJobsAsync(false, ct);
return await GetReadyJobsAsync(false);
}
@@ -59,8 +61,9 @@ namespace AyaNova.Biz
/// Get a non tracked list of jobs filtered by exclusivity
/// </summary>
/// <returns></returns>
private static async Task<List<OpsJob>> GetReadyJobsAsync(bool exclusiveOnly, AyContext ct)
private static async Task<List<OpsJob>> GetReadyJobsAsync(bool exclusiveOnly)
{
var ct = ServiceProviderProvider.DBContext;
var ret = await ct.OpsJob
.AsNoTracking()
.Where(z => z.StartAfter < System.DateTime.UtcNow && z.Exclusive == exclusiveOnly && z.JobStatus == JobStatus.Sleeping)
@@ -76,8 +79,9 @@ namespace AyaNova.Biz
/// could be running or sleeping
/// </summary>
/// <returns></returns>
internal static async Task<List<OpsJob>> GetAllSleepingOrRunningJobsAsync(AyContext ct)
internal static async Task<List<OpsJob>> GetAllSleepingOrRunningJobsAsync()
{
var ct = ServiceProviderProvider.DBContext;
var ret = await ct.OpsJob
.AsNoTracking()
.Where(z => z.JobStatus == JobStatus.Sleeping || z.JobStatus == JobStatus.Running)
@@ -92,8 +96,9 @@ namespace AyaNova.Biz
/// Get a non tracked list of all jobs for a JobType
/// </summary>
/// <returns></returns>
internal static async Task<List<OpsJob>> GetAllJobsForJobTypeAsync(AyContext ct, JobType jobType)
internal static async Task<List<OpsJob>> GetAllJobsForJobTypeAsync(JobType jobType)
{
var ct = ServiceProviderProvider.DBContext;
var ret = await ct.OpsJob
.AsNoTracking()
.Where(z => z.JobType == jobType)
@@ -108,8 +113,9 @@ namespace AyaNova.Biz
/// Get a non tracked list of all jobs that are status running but have no last activity for XX HOURS
/// </summary>
/// <returns></returns>
internal static async Task<List<OpsJob>> GetPotentiallyDeadRunningJobsAsync(AyContext ct)
internal static async Task<List<OpsJob>> GetPotentiallyDeadRunningJobsAsync()
{
var ct = ServiceProviderProvider.DBContext;
var ret = await ct.OpsJob
.AsNoTracking()
.Where(z => z.JobStatus == JobStatus.Sleeping || z.JobStatus == JobStatus.Running)
@@ -124,8 +130,9 @@ namespace AyaNova.Biz
/// Get a count of all jobs for a JobStatus
/// </summary>
/// <returns></returns>
internal static async Task<long> GetCountForJobStatusAsync(AyContext ct, JobStatus jobStatus)
internal static async Task<long> GetCountForJobStatusAsync(JobStatus jobStatus)
{
var ct = ServiceProviderProvider.DBContext;
var ret = await ct.OpsJob
.Where(z => z.JobStatus == jobStatus)
.LongCountAsync();
@@ -138,10 +145,11 @@ namespace AyaNova.Biz
/// Add a new job to the database
/// </summary>
/// <param name="newJob"></param>
/// <param name="ct"></param>
/// <returns></returns>
internal static async Task<OpsJob> AddJobAsync(OpsJob newJob, AyContext ct)
internal static async Task<OpsJob> AddJobAsync(OpsJob newJob)
{
var ct = ServiceProviderProvider.DBContext;
await ct.OpsJob.AddAsync(newJob);
await ct.SaveChangesAsync();
return newJob;
@@ -151,110 +159,57 @@ namespace AyaNova.Biz
/// Remove any jobs or logs for the object in question
/// </summary>
/// <param name="ayObj"></param>
/// <param name="ct"></param>
///
internal static async Task DeleteJobsForObjectAsync(AyaTypeId ayObj, AyContext ct)
{
//Get a list of all jobid's for the object passed in
List<OpsJob> jobsForObject = await GetJobsForObjectAsync(ayObj, ct);
//short circuit
List<OpsJob> jobsForObject = await GetJobsForObjectAsync(ayObj);
if (jobsForObject.Count == 0)
return;
using (var transaction = await ct.Database.BeginTransactionAsync())
foreach (OpsJob jobToBeDeleted in jobsForObject)
{
try
{
foreach (OpsJob jobToBeDeleted in jobsForObject)
{
await RemoveJobAndLogsAsync(ct, jobToBeDeleted.GId);
}
// Commit transaction if all commands succeed, transaction will auto-rollback
// when disposed if either commands fails
await transaction.CommitAsync();
}
catch (Exception ex)
{
throw ex;
}
await RemoveJobAndLogsAsync(jobToBeDeleted.GId);
}
}
/// <summary>
/// Remove job and logs for that job
/// </summary>
/// <param name="jobId"></param>
/// <param name="ct"></param>
internal static async Task DeleteJobAndLogAsync(Guid jobId, AyContext ct)
{
using (var transaction = await ct.Database.BeginTransactionAsync())
{
try
{
await RemoveJobAndLogsAsync(ct, jobId);
// Commit transaction if all commands succeed, transaction will auto-rollback
// when disposed if either commands fails
await transaction.CommitAsync();
}
catch (Exception ex)
{
throw ex;
}
}
}
/// <summary>
/// REmove the job and it's logs
/// </summary>
/// <param name="ct"></param>
/// </summary>
/// <param name="jobIdToBeDeleted"></param>
private static async Task RemoveJobAndLogsAsync(AyContext ct, Guid jobIdToBeDeleted)
internal static async Task RemoveJobAndLogsAsync(Guid jobIdToBeDeleted)
{
// //delete logs
// await ct.Database.ExecuteSqlCommandAsync("delete from aopsjoblog where jobid = {0}", new object[] { jobIdToBeDeleted });
// //delete the job
// await ct.Database.ExecuteSqlCommandAsync("delete from aopsjob where gid = {0}", new object[] { jobIdToBeDeleted });
//delete logs
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjoblog where jobid = {jobIdToBeDeleted}");
//delete the job
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjob where gid = {jobIdToBeDeleted}");
var ct = ServiceProviderProvider.DBContext;
using (var transaction = await ct.Database.BeginTransactionAsync())
{
try
{
//delete logs
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjoblog where jobid = {jobIdToBeDeleted}");
//delete the job
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjob where gid = {jobIdToBeDeleted}");
// Commit transaction if all commands succeed, transaction will auto-rollback
// when disposed if either commands fails
await transaction.CommitAsync();
}
catch (Exception ex)
{
throw ex;
}
}
}
/// <summary>
/// Make a log entry for a job
/// (no context version)
/// </summary>
/// <param name="jobId">(NOTE: Guid.empty indicates internal job)</param>
/// <param name="statusText"></param>
///
internal static async Task<OpsJobLog> LogJobAsync(Guid jobId, string statusText)
{
return await LogJobAsync(jobId, statusText, null);
}
/// <summary>
/// Make a log entry for a job
/// </summary>
/// <param name="jobId">(NOTE: Guid.empty indicates internal job)</param>
/// <param name="statusText"></param>
/// <param name="ct"></param>
internal static async Task<OpsJobLog> LogJobAsync(Guid jobId, string statusText, AyContext ct)
{
if (ct == null)
ct = ServiceProviderProvider.DBContext;
var ct = ServiceProviderProvider.DBContext;
if (string.IsNullOrWhiteSpace(statusText))
statusText = "No status provided";
OpsJobLog newObj = new OpsJobLog();
@@ -270,10 +225,10 @@ namespace AyaNova.Biz
/// Update the status of a job
/// </summary>
/// <param name="jobId"></param>
/// <param name="newStatus"></param>
/// <param name="ct"></param>
internal static async Task<OpsJob> UpdateJobStatusAsync(Guid jobId, JobStatus newStatus, AyContext ct)
/// <param name="newStatus"></param>
internal static async Task<OpsJob> UpdateJobStatusAsync(Guid jobId, JobStatus newStatus)
{
var ct = ServiceProviderProvider.DBContext;
var oFromDb = await ct.OpsJob.SingleOrDefaultAsync(z => z.GId == jobId);
if (oFromDb == null) return null;
oFromDb.JobStatus = newStatus;
@@ -289,37 +244,70 @@ namespace AyaNova.Biz
/// Process all jobs (stock jobs and those found in operations table)
/// </summary>
/// <returns></returns>
internal static async Task ProcessJobsAsync(AyContext ct, AyaNova.Api.ControllerHelpers.ApiServerState serverState)
internal static async Task ProcessJobsAsync(CancellationToken ctoken)
{
if (ActivelyProcessing)
{
System.Diagnostics.Debug.WriteLine("ProcessJobs called but actively processing other jobs so returning");
log.LogTrace("ProcessJobs called but actively processing other jobs so returning");
return;
}
ActivelyProcessing = true;
try
{
ctoken.ThrowIfCancellationRequested();
//Sweep jobs table
System.Diagnostics.Debug.WriteLine($"JobsBiz processing sweeper");
await CoreJobSweeper.DoSweepAsync(ctoken);//run exclusively
//BIZOBJECT DYNAMIC JOBS
//get a list of exclusive jobs that are due to happen
//Call into each item in turn
List<OpsJob> exclusiveJobs = await GetReadyJobsExclusiveOnlyAsync(ct);
List<OpsJob> exclusiveJobs = await GetReadyJobsExclusiveOnlyAsync();
foreach (OpsJob j in exclusiveJobs)
{
try
{
await ProcessJobAsync(j, ct);
ctoken.ThrowIfCancellationRequested();
System.Diagnostics.Debug.WriteLine($"JobsBiz processing exclusive biz job {j.Name}");
await ProcessJobAsync(j);
}
catch (Exception ex)
{
log.LogError(ex, $"ProcessJobs::Exclusive -> job {j.Name} failed with exception");
await LogJobAsync(j.GId, "Job failed with errors:", ct);
await LogJobAsync(j.GId, ExceptionUtil.ExtractAllExceptionMessages(ex), ct);
await UpdateJobStatusAsync(j.GId, JobStatus.Failed, ct);
await LogJobAsync(j.GId, "Job failed with errors:");
await LogJobAsync(j.GId, ExceptionUtil.ExtractAllExceptionMessages(ex));
await UpdateJobStatusAsync(j.GId, JobStatus.Failed);
}
}
//Get a list of non-exlusive jobs that are due
ctoken.ThrowIfCancellationRequested();
System.Diagnostics.Debug.WriteLine($"JobsBiz processing exclusive license check");
//License check
long CurrentActiveCount = await UserBiz.ActiveCountAsync();
long LicensedUserCount = AyaNova.Core.License.ActiveKey.ActiveNumber;
// log.LogInformation("JobsBiz::Checking license active count");
if (CurrentActiveCount > LicensedUserCount)
{
var msg = $"E1020 - Active count exceeded capacity";
ServiceProviderProvider.ServerState.SetSystemLock(msg);
log.LogCritical(msg);
return;
}
ctoken.ThrowIfCancellationRequested();
//backup
System.Diagnostics.Debug.WriteLine($"JobsBiz processing backup");
await CoreJobBackup.DoWorkAsync();//sb exclusive
System.Diagnostics.Debug.WriteLine($"JobsBiz processing metrics snapshotter");
///////////////////////////////////////
//NON-EXCLUSIVE JOBS
//
//LOOKAT: Parallelize / background this block
//http://www.dotnetcurry.com/dotnet/1360/concurrent-programming-dotnet-core
@@ -329,42 +317,31 @@ namespace AyaNova.Biz
//This area may turn out to need a re-write in future, but I think it might only involve this block and ProcessJobAsync
//the actual individual objects that are responsible for jobs will likely not need a signature rewrite or anything (I hope)
//For now I'z hoping that no job will be so slow that it can hold up all the other jobs indefinitely.
List<OpsJob> sharedJobs = await GetReadyJobsNotExlusiveOnlyAsync(ct);
ctoken.ThrowIfCancellationRequested();
List<OpsJob> sharedJobs = await GetReadyJobsNotExlusiveOnlyAsync();
foreach (OpsJob j in sharedJobs)
{
try
{
await ProcessJobAsync(j, ct);
ctoken.ThrowIfCancellationRequested();
System.Diagnostics.Debug.WriteLine($"JobsBiz processing NON-exclusive biz job {j.Name}");
// Task.Run(() => FireAway());
Task.Run(() => ProcessJobAsync(j), ctoken);
// await ProcessJobAsync(j, ct);
}
catch (Exception ex)
{
log.LogError(ex, $"ProcessJobs::Shared -> job {j.Name} failed with exception");
await LogJobAsync(j.GId, "Job failed with errors:", ct);
await LogJobAsync(j.GId, ExceptionUtil.ExtractAllExceptionMessages(ex), ct);
await UpdateJobStatusAsync(j.GId, JobStatus.Failed, ct);
await LogJobAsync(j.GId, "Job failed with errors:");
await LogJobAsync(j.GId, ExceptionUtil.ExtractAllExceptionMessages(ex));
await UpdateJobStatusAsync(j.GId, JobStatus.Failed);
}
}
//STOCK JOBS
//Sweep jobs table
await CoreJobSweeper.DoSweepAsync(ct);
//License check
long CurrentActiveCount = await UserBiz.ActiveCountAsync();
long LicensedUserCount = AyaNova.Core.License.ActiveKey.ActiveNumber;
// log.LogInformation("JobsBiz::Checking license active count");
if (CurrentActiveCount > LicensedUserCount)
{
var msg = $"E1020 - Active count exceeded capacity";
serverState.SetSystemLock(msg);
log.LogCritical(msg);
return;
}
//STOCK JOBS
//backup
await CoreJobBackup.DoWorkAsync(ct);
await CoreJobMetricsSnapshot.DoJobAsync(ct);
//Notifications
@@ -377,6 +354,7 @@ namespace AyaNova.Biz
finally
{
ActivelyProcessing = false;
System.Diagnostics.Debug.WriteLine($"JobsBiz in Finally - completed run");
}
}
@@ -384,15 +362,14 @@ namespace AyaNova.Biz
/// <summary>
/// Process a job by calling into it's biz object
/// </summary>
/// <param name="job"></param>
/// <param name="ct"></param>
/// <param name="job"></param>
/// <returns></returns>
internal static async Task ProcessJobAsync(OpsJob job, AyContext ct)
internal static async Task ProcessJobAsync(OpsJob job)
{
var JobDescription = $"{job.Name} {job.JobType.ToString()}";
if (job.SubType != JobSubType.NotSet)
JobDescription += $":{job.SubType}";
await LogJobAsync(job.GId, $"Process job \"{JobDescription}\"", ct);
await LogJobAsync(job.GId, $"Process job \"{JobDescription}\"");
log.LogDebug($"ProcessJobAsync -> Processing job {JobDescription}");
IJobObject o = null;
@@ -400,19 +377,19 @@ namespace AyaNova.Biz
{
case JobType.Backup:
//This is called when on demand only, normal backups are processed above with normal system jobs
await CoreJobBackup.DoWorkAsync(ct, true);
await UpdateJobStatusAsync(job.GId, JobStatus.Completed, ct);
await CoreJobBackup.DoWorkAsync(true);
await UpdateJobStatusAsync(job.GId, JobStatus.Completed);
break;
case JobType.TestJob:
o = (IJobObject)BizObjectFactory.GetBizObject(AyaType.ServerJob, ct);
o = (IJobObject)BizObjectFactory.GetBizObject(AyaType.ServerJob);
break;
case JobType.SeedTestData:
o = (IJobObject)BizObjectFactory.GetBizObject(AyaType.TrialSeeder, ct);
o = (IJobObject)BizObjectFactory.GetBizObject(AyaType.TrialSeeder);
break;
case JobType.BulkCoreBizObjectOperation:
//bulk op, hand off to biz object to deal with
//note, convention is that there is an idList in job.jobinfo json if preselected else it's all objects of type
o = (IJobObject)BizObjectFactory.GetBizObject(job.ObjectType, ct);
o = (IJobObject)BizObjectFactory.GetBizObject(job.ObjectType);
break;
default:
throw new System.NotSupportedException($"ProcessJobAsync type {job.JobType.ToString()} is not supported");

5
server/AyaNova/biz/Search.cs
View File

@@ -246,8 +246,7 @@ namespace AyaNova.Biz
#endregion dosearch
#region Get info (excerpt)
public static async Task<string> GetInfoAsync(AyContext ct, long translationId,
AuthorizationRoles currentUserRoles, long userId, string phrase, int max, AyaType ayaType, long id)
public static async Task<string> GetInfoAsync(long translationId, AuthorizationRoles currentUserRoles, long userId, string phrase, int max, AyaType ayaType, long id)
{
//escape literal percentage signs first just in case they are searching for 50% off or something
//https://www.postgresql.org/docs/current/functions-matching.html#FUNCTIONS-LIKE
@@ -262,7 +261,7 @@ namespace AyaNova.Biz
PhraseItems.ToArray();
//get text
ISearchAbleObject o = (ISearchAbleObject)BizObjectFactory.GetBizObject(ayaType, ct, userId, currentUserRoles);
ISearchAbleObject o = (ISearchAbleObject)BizObjectFactory.GetBizObject(ayaType, userId, currentUserRoles);
//get extract
var searchParams = await o.GetSearchResultSummary(id);

14
server/AyaNova/biz/WorkOrderBiz.cs
View File

@@ -2441,8 +2441,8 @@ namespace AyaNova.Biz
private async Task ProcessBulkJobAsync(OpsJob job)
{
await JobsBiz.UpdateJobStatusAsync(job.GId, JobStatus.Running, ct);
await JobsBiz.LogJobAsync(job.GId, $"Bulk job {job.SubType} started...", ct);
await JobsBiz.UpdateJobStatusAsync(job.GId, JobStatus.Running);
await JobsBiz.LogJobAsync(job.GId, $"Bulk job {job.SubType} started...");
List<long> idList = new List<long>();
long ProcessedObjectCount = 0;
JObject jobData = JObject.Parse(job.JobInfo);
@@ -2475,7 +2475,7 @@ namespace AyaNova.Biz
{
o = await PutWorkOrderGraphItem(job.ObjectType, o);
if (o == null)
await JobsBiz.LogJobAsync(job.GId, $"Error processing item {id}: {GetErrorsAsString()}", ct);
await JobsBiz.LogJobAsync(job.GId, $"Error processing item {id}: {GetErrorsAsString()}");
else
ProcessedObjectCount++;
}
@@ -2484,12 +2484,12 @@ namespace AyaNova.Biz
}
catch (Exception ex)
{
await JobsBiz.LogJobAsync(job.GId, $"Error processing item {id}", ct);
await JobsBiz.LogJobAsync(job.GId, ExceptionUtil.ExtractAllExceptionMessages(ex), ct);
await JobsBiz.LogJobAsync(job.GId, $"Error processing item {id}");
await JobsBiz.LogJobAsync(job.GId, ExceptionUtil.ExtractAllExceptionMessages(ex));
}
}
await JobsBiz.LogJobAsync(job.GId, $"Bulk job {job.SubType} processed {ProcessedObjectCount} of {idList.Count}", ct);
await JobsBiz.UpdateJobStatusAsync(job.GId, JobStatus.Completed, ct);
await JobsBiz.LogJobAsync(job.GId, $"Bulk job {job.SubType} processed {ProcessedObjectCount} of {idList.Count}");
await JobsBiz.UpdateJobStatusAsync(job.GId, JobStatus.Completed);
}

33
server/AyaNova/generator/CoreJobMetricsSnapshot.cs
View File

@@ -27,17 +27,18 @@ namespace AyaNova.Biz
#if(DEBUG)
private static TimeSpan tsOneMinute = new TimeSpan(0, 0, 10);
#else
#else
private static TimeSpan tsOneMinute = new TimeSpan(0, 1, 0);
#endif
#endif
private static TimeSpan tsOneHour = new TimeSpan(1, 0, 0);
private static TimeSpan ts24Hours = new TimeSpan(24, 0, 0);
////////////////////////////////////////////////////////////////////////////////////////////////
// DoAsync
//
public static async Task DoJobAsync(AyContext ct)
public static void DoJob()
{
#region worksheet
//https://github.com/sebastienros/memoryleak/blob/master/src/MemoryLeak/MemoryLeak/Controllers/DiagnosticsController.cs
// //DATA TYPES .net to postgres map
//http://www.npgsql.org/doc/types/basic.html
@@ -153,6 +154,7 @@ from generate_series(1, 525600) s(i)
,...);
*/
#endregion worksheet
//Nothing is gathered less than one minute frequency
if (!DateUtil.IsAfterDuration(_lastSnapshot, tsOneMinute))
@@ -196,12 +198,15 @@ from generate_series(1, 525600) s(i)
var Gen2 = GC.CollectionCount(2);//integer
var CPU = _cpu;// double precision
//write to db
MetricMM mm = new MetricMM(Allocated, WorkingSet, PrivateBytes, Gen0, Gen1, Gen2, CPU);
await ct.MetricMM.AddAsync(mm);
await ct.SaveChangesAsync();
System.Diagnostics.Debug.WriteLine($"MM Snapshot, cpu: {CPU}");
using (AyContext ct = ServiceProviderProvider.DBContext)
{
//write to db
MetricMM mm = new MetricMM(Allocated, WorkingSet, PrivateBytes, Gen0, Gen1, Gen2, CPU);
ct.MetricMM.Add(mm);
ct.SaveChanges();
System.Diagnostics.Debug.WriteLine("MM SAVED");
}
/////////////////////////////////////////////
@@ -270,17 +275,17 @@ from generate_series(1, 525600) s(i)
//
DateTime ClearDate = DateTime.UtcNow - tsDataRetention;
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from ametricmm where t > {ClearDate}");
using (AyContext ct = ServiceProviderProvider.DBContext)
{
ct.Database.ExecuteSqlInterpolated($"delete from ametricmm where t < {ClearDate}");
}
}
_lastSnapshot = now;
{
DateTime ClearDate = DateTime.UtcNow - tsDataRetention;
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from ametricmm where t > {ClearDate}");
}
}

63
server/AyaNova/generator/CoreJobSweeper.cs
View File

@@ -4,7 +4,7 @@ using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using AyaNova.Models;
using System.Threading;
namespace AyaNova.Biz
{
@@ -28,44 +28,49 @@ namespace AyaNova.Biz
////////////////////////////////////////////////////////////////////////////////////////////////
// DoSweep
//
public static async Task DoSweepAsync(AyContext ct)
public static async Task DoSweepAsync(CancellationToken ctoken)
{
ctoken.ThrowIfCancellationRequested();
//This will get triggered roughly every minute, but we don't want to sweep that frequently
if (DateTime.UtcNow - lastSweep < SWEEP_EVERY_INTERVAL)
return;
log.LogTrace("Sweep starting");
using (AyContext ct = AyaNova.Util.ServiceProviderProvider.DBContext)
{
//SWEEP SUCCESSFUL JOBS
//calculate cutoff to delete
DateTime dtDeleteCutoff = DateTime.UtcNow - SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Completed);
//SWEEP SUCCESSFUL JOBS
//calculate cutoff to delete
DateTime dtDeleteCutoff = DateTime.UtcNow - SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Completed, ctoken);
//SWEEP FAILED JOBS
//calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Failed);
//KILL STUCK JOBS
//calculate cutoff to delete
DateTime dtRunningDeadline = DateTime.UtcNow - RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN;
await killStuckJobsAsync(ct, dtRunningDeadline);
//SWEEP INTERNAL JOB LOG
//calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - INTERNAL_JOBS_LOGS_DELETE_AFTER_THIS_TIMESPAN;
await SweepInternalJobsLogsAsync(ct, dtDeleteCutoff);
ctoken.ThrowIfCancellationRequested();
//SWEEP FAILED JOBS
//calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Failed, ctoken);
ctoken.ThrowIfCancellationRequested();
//KILL STUCK JOBS
//calculate cutoff to delete
DateTime dtRunningDeadline = DateTime.UtcNow - RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN;
await killStuckJobsAsync(ct, dtRunningDeadline);
ctoken.ThrowIfCancellationRequested();
//SWEEP INTERNAL JOB LOG
//calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - INTERNAL_JOBS_LOGS_DELETE_AFTER_THIS_TIMESPAN;
await SweepInternalJobsLogsAsync(ct, dtDeleteCutoff);
ctoken.ThrowIfCancellationRequested();
}
lastSweep = DateTime.UtcNow;
}
private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus)
private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus, CancellationToken ctoken)//AyContext ct,
{
// AyContext ct = ServiceProviderProvider.DBContext;
//Get the deleteable succeeded jobs list
var jobs = await ct.OpsJob
.AsNoTracking()
@@ -79,11 +84,12 @@ namespace AyaNova.Biz
{
try
{
await JobsBiz.DeleteJobAndLogAsync(j.GId, ct);
ctoken.ThrowIfCancellationRequested();
await JobsBiz.RemoveJobAndLogsAsync(j.GId);
}
catch (Exception ex)
{
log.LogError(ex, "sweepAsync exception calling JobsBiz.DeleteJobAndLogAsync");
log.LogError(ex, "sweepAsync exception calling JobsBiz.RemoveJobAndLogsAsync");
//for now just throw it but this needs to be removed when logging added and better handling
throw (ex);
}
@@ -93,12 +99,13 @@ namespace AyaNova.Biz
/// <summary>
/// Kill jobs that have been stuck in "running" state for too long
/// </summary>
/// <param name="ct"></param>
/// </summary>
/// <param name="dtRunningDeadline"></param>
/// <returns></returns>
private static async Task killStuckJobsAsync(AyContext ct, DateTime dtRunningDeadline)
{
// AyContext ct = ServiceProviderProvider.DBContext;
//Get the deleteable succeeded jobs list
var jobs = await ct.OpsJob
.AsNoTracking()
@@ -120,7 +127,7 @@ namespace AyaNova.Biz
private static async Task SweepInternalJobsLogsAsync(AyContext ct, DateTime dtDeleteCutoff)
{
// AyContext ct = ServiceProviderProvider.DBContext;
//Get the deleteable list (this is for reporting, could easily just do it in one go)
var logs = await ct.OpsJobLog
.AsNoTracking()

51
server/AyaNova/generator/Generate.cs
View File

@@ -6,6 +6,7 @@ using Microsoft.Extensions.DependencyInjection;
using AyaNova.Models;
using AyaNova.Api.ControllerHelpers;
using AyaNova.Biz;
using AyaNova.Util;
namespace AyaNova.Generator
{
@@ -27,6 +28,7 @@ namespace AyaNova.Generator
// private readonly ApiServerState serverState;
private readonly IServiceProvider provider;
private const int MAXIMUM_TIME_ALLOWED_FOR_PROCESSING_ALL_JOBS = 1 * 60 * 1000;//1 minutes TEST TEST TEST #####
#if(DEBUG)
private const int GENERATE_SECONDS = 5;
@@ -53,7 +55,11 @@ namespace AyaNova.Generator
/*
todo: improve this
it should timeout: https://stackoverflow.com/questions/23476576/cancellationtoken-timeout-vs-task-delay-and-timeout
it should never stop running no matter what unless teh server shuts down
*/
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
@@ -73,32 +79,37 @@ namespace AyaNova.Generator
{
log.LogDebug($"GeneratorService task doing background work.");
using (IServiceScope scope = provider.CreateScope())
ApiServerState serverState = ServiceProviderProvider.ServerState;
//=================================================================
try
{
AyContext ct = scope.ServiceProvider.GetRequiredService<AyContext>();
ApiServerState serverState = scope.ServiceProvider.GetRequiredService<ApiServerState>();
//=================================================================
try
if (!serverState.IsOpen)
{
if (!serverState.IsOpen)
{
log.LogDebug($"GeneratorService: ServerState is closed returning without processing jobs, will try again next iteration");
}
else
{
await JobsBiz.ProcessJobsAsync(ct, serverState);
}
log.LogDebug($"GeneratorService: ServerState is closed returning without processing jobs, will try again next iteration");
}
catch (Exception ex)
else
{
log.LogError(ex, "Generate::ProcessJobs result in exception error ");
System.Diagnostics.Debug.WriteLine($"### GENERATE calling JobsBiz.ProcessJobs");
//Before anything capture metrics
// await CoreJobMetricsSnapshot.DoJobAsync(ct);
//Task.Run(() => CoreJobMetricsSnapshot.DoJob());
//deliberately calling this non-async
//it needs to be fast and efficient
CoreJobMetricsSnapshot.DoJob();
//TODO: this should be big timeout and then inside the process jobs each job has it's own timeout
await TaskUtil.WithTimeoutAfterStart(ctoken => JobsBiz.ProcessJobsAsync(ctoken), TimeSpan.FromMilliseconds(MAXIMUM_TIME_ALLOWED_FOR_PROCESSING_ALL_JOBS));
// await JobsBiz.ProcessJobsAsync(ct, serverState);
System.Diagnostics.Debug.WriteLine($"### GENERATE BACK FROM calling JobsBiz.ProcessJobs");
}
//=================================================================
}
catch (Exception ex)
{
log.LogError(ex, "Generate::ProcessJobs result in exception error ");
}
//=================================================================
}
await Task.Delay((GENERATE_SECONDS * 1000), stoppingToken);
justStarted = false;

8
server/AyaNova/util/ServiceProviderProvider.cs
View File

@@ -58,6 +58,14 @@ namespace AyaNova.Util
}
}
internal static AyaNova.Api.ControllerHelpers.ApiServerState ServerState
{
get
{
return Scope.ServiceProvider.GetRequiredService<AyaNova.Api.ControllerHelpers.ApiServerState>();
}
}
// https://docs.microsoft.com/en-us/aspnet/core/fundamentals/http-requests?view=aspnetcore-3.1
internal static System.Net.Http.IHttpClientFactory HttpClientFactory
{