admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

much fuckery

Browse Source
This commit is contained in:
John Cardinal
2020-05-25 20:34:11 +00:00
parent d977728a22
commit e90dbe0f71
9 changed files with 218 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/AyaNova/Controllers/BackupController.cs
View File

@@ -68,7 +68,7 @@ namespace AyaNova.Api.Controllers
j.JobType = JobType.Backup; j.JobType = JobType.Backup;
j.SubType = JobSubType.NotSet; j.SubType = JobSubType.NotSet;
j.Exclusive = true; j.Exclusive = true;
await JobsBiz.AddJobAsync(j, ct); await JobsBiz.AddJobAsync(j);
await EventLogProcessor.LogEventToDatabaseAsync(new Event(UserIdFromContext.Id(HttpContext.Items), 0, AyaType.ServerJob, AyaEvent.Created, JobName), ct); await EventLogProcessor.LogEventToDatabaseAsync(new Event(UserIdFromContext.Id(HttpContext.Items), 0, AyaType.ServerJob, AyaEvent.Created, JobName), ct);
return Accepted(new { JobId = j.GId }); return Accepted(new { JobId = j.GId });
} }

3
server/AyaNova/biz/BizObjectFactory.cs
View File

@@ -20,8 +20,9 @@ namespace AyaNova.Biz
//Returns the biz object class that corresponds to the type presented //Returns the biz object class that corresponds to the type presented
//Used by SEARCH and objects with JOBS //Used by SEARCH and objects with JOBS
internal static BizObject GetBizObject(AyaType ayaType, AyContext dbcontext, long userId = 1, AuthorizationRoles roles = AuthorizationRoles.All) internal static BizObject GetBizObject(AyaType ayaType, long userId = 1, AuthorizationRoles roles = AuthorizationRoles.All)
{ {
var dbcontext = ServiceProviderProvider.DBContext;
switch (ayaType) switch (ayaType)
{ {
//CoreBizObject add here //CoreBizObject add here

239
server/AyaNova/biz/JobsBiz.cs
View File

@@ -1,6 +1,7 @@
using System; using System;
using System.Linq; using System.Linq;
using System.Threading.Tasks; using System.Threading.Tasks;
using System.Threading;
using System.Collections.Generic; using System.Collections.Generic;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
@@ -22,10 +23,11 @@ namespace AyaNova.Biz
/// Get a non tracked list of jobs for an object /// Get a non tracked list of jobs for an object
/// </summary> /// </summary>
/// <param name="ayObj"></param> /// <param name="ayObj"></param>
/// <param name="ct"></param> ///
/// <returns></returns> /// <returns></returns>
internal static async Task<List<OpsJob>> GetJobsForObjectAsync(AyaTypeId ayObj, AyContext ct) internal static async Task<List<OpsJob>> GetJobsForObjectAsync(AyaTypeId ayObj)
{ {
var ct = ServiceProviderProvider.DBContext;
return await ct.OpsJob return await ct.OpsJob
.AsNoTracking() .AsNoTracking()
.Where(z => z.ObjectId == ayObj.ObjectId && z.ObjectType == ayObj.ObjectType) .Where(z => z.ObjectId == ayObj.ObjectId && z.ObjectType == ayObj.ObjectType)
@@ -39,9 +41,9 @@ namespace AyaNova.Biz
/// Get a non tracked list of jobs that are ready to process and exclusive only /// Get a non tracked list of jobs that are ready to process and exclusive only
/// </summary> /// </summary>
/// <returns></returns> /// <returns></returns>
internal static async Task<List<OpsJob>> GetReadyJobsExclusiveOnlyAsync(AyContext ct) internal static async Task<List<OpsJob>> GetReadyJobsExclusiveOnlyAsync()
{ {
return await GetReadyJobsAsync(true, ct); return await GetReadyJobsAsync(true);
} }
@@ -49,9 +51,9 @@ namespace AyaNova.Biz
/// Get a non tracked list of jobs that are ready to process and exclusive only /// Get a non tracked list of jobs that are ready to process and exclusive only
/// </summary> /// </summary>
/// <returns></returns> /// <returns></returns>
internal static async Task<List<OpsJob>> GetReadyJobsNotExlusiveOnlyAsync(AyContext ct) internal static async Task<List<OpsJob>> GetReadyJobsNotExlusiveOnlyAsync()
{ {
return await GetReadyJobsAsync(false, ct); return await GetReadyJobsAsync(false);
} }
@@ -59,8 +61,9 @@ namespace AyaNova.Biz
/// Get a non tracked list of jobs filtered by exclusivity /// Get a non tracked list of jobs filtered by exclusivity
/// </summary> /// </summary>
/// <returns></returns> /// <returns></returns>
private static async Task<List<OpsJob>> GetReadyJobsAsync(bool exclusiveOnly, AyContext ct) private static async Task<List<OpsJob>> GetReadyJobsAsync(bool exclusiveOnly)
{ {
var ct = ServiceProviderProvider.DBContext;
var ret = await ct.OpsJob var ret = await ct.OpsJob
.AsNoTracking() .AsNoTracking()
.Where(z => z.StartAfter < System.DateTime.UtcNow && z.Exclusive == exclusiveOnly && z.JobStatus == JobStatus.Sleeping) .Where(z => z.StartAfter < System.DateTime.UtcNow && z.Exclusive == exclusiveOnly && z.JobStatus == JobStatus.Sleeping)
@@ -76,8 +79,9 @@ namespace AyaNova.Biz
/// could be running or sleeping /// could be running or sleeping
/// </summary> /// </summary>
/// <returns></returns> /// <returns></returns>
internal static async Task<List<OpsJob>> GetAllSleepingOrRunningJobsAsync(AyContext ct) internal static async Task<List<OpsJob>> GetAllSleepingOrRunningJobsAsync()
{ {
var ct = ServiceProviderProvider.DBContext;
var ret = await ct.OpsJob var ret = await ct.OpsJob
.AsNoTracking() .AsNoTracking()
.Where(z => z.JobStatus == JobStatus.Sleeping || z.JobStatus == JobStatus.Running) .Where(z => z.JobStatus == JobStatus.Sleeping || z.JobStatus == JobStatus.Running)
@@ -92,8 +96,9 @@ namespace AyaNova.Biz
/// Get a non tracked list of all jobs for a JobType /// Get a non tracked list of all jobs for a JobType
/// </summary> /// </summary>
/// <returns></returns> /// <returns></returns>
internal static async Task<List<OpsJob>> GetAllJobsForJobTypeAsync(AyContext ct, JobType jobType) internal static async Task<List<OpsJob>> GetAllJobsForJobTypeAsync(JobType jobType)
{ {
var ct = ServiceProviderProvider.DBContext;
var ret = await ct.OpsJob var ret = await ct.OpsJob
.AsNoTracking() .AsNoTracking()
.Where(z => z.JobType == jobType) .Where(z => z.JobType == jobType)
@@ -108,8 +113,9 @@ namespace AyaNova.Biz
/// Get a non tracked list of all jobs that are status running but have no last activity for XX HOURS /// Get a non tracked list of all jobs that are status running but have no last activity for XX HOURS
/// </summary> /// </summary>
/// <returns></returns> /// <returns></returns>
internal static async Task<List<OpsJob>> GetPotentiallyDeadRunningJobsAsync(AyContext ct) internal static async Task<List<OpsJob>> GetPotentiallyDeadRunningJobsAsync()
{ {
var ct = ServiceProviderProvider.DBContext;
var ret = await ct.OpsJob var ret = await ct.OpsJob
.AsNoTracking() .AsNoTracking()
.Where(z => z.JobStatus == JobStatus.Sleeping || z.JobStatus == JobStatus.Running) .Where(z => z.JobStatus == JobStatus.Sleeping || z.JobStatus == JobStatus.Running)
@@ -124,8 +130,9 @@ namespace AyaNova.Biz
/// Get a count of all jobs for a JobStatus /// Get a count of all jobs for a JobStatus
/// </summary> /// </summary>
/// <returns></returns> /// <returns></returns>
internal static async Task<long> GetCountForJobStatusAsync(AyContext ct, JobStatus jobStatus) internal static async Task<long> GetCountForJobStatusAsync(JobStatus jobStatus)
{ {
var ct = ServiceProviderProvider.DBContext;
var ret = await ct.OpsJob var ret = await ct.OpsJob
.Where(z => z.JobStatus == jobStatus) .Where(z => z.JobStatus == jobStatus)
.LongCountAsync(); .LongCountAsync();
@@ -138,10 +145,11 @@ namespace AyaNova.Biz
/// Add a new job to the database /// Add a new job to the database
/// </summary> /// </summary>
/// <param name="newJob"></param> /// <param name="newJob"></param>
/// <param name="ct"></param>
/// <returns></returns> /// <returns></returns>
internal static async Task<OpsJob> AddJobAsync(OpsJob newJob, AyContext ct) internal static async Task<OpsJob> AddJobAsync(OpsJob newJob)
{ {
var ct = ServiceProviderProvider.DBContext;
await ct.OpsJob.AddAsync(newJob); await ct.OpsJob.AddAsync(newJob);
await ct.SaveChangesAsync(); await ct.SaveChangesAsync();
return newJob; return newJob;
@@ -151,110 +159,57 @@ namespace AyaNova.Biz
/// Remove any jobs or logs for the object in question /// Remove any jobs or logs for the object in question
/// </summary> /// </summary>
/// <param name="ayObj"></param> /// <param name="ayObj"></param>
/// <param name="ct"></param> ///
internal static async Task DeleteJobsForObjectAsync(AyaTypeId ayObj, AyContext ct) internal static async Task DeleteJobsForObjectAsync(AyaTypeId ayObj, AyContext ct)
{ {
//Get a list of all jobid's for the object passed in //Get a list of all jobid's for the object passed in
List<OpsJob> jobsForObject = await GetJobsForObjectAsync(ayObj, ct); List<OpsJob> jobsForObject = await GetJobsForObjectAsync(ayObj);
//short circuit
if (jobsForObject.Count == 0) if (jobsForObject.Count == 0)
return; return;
foreach (OpsJob jobToBeDeleted in jobsForObject)
using (var transaction = await ct.Database.BeginTransactionAsync())
{ {
try await RemoveJobAndLogsAsync(jobToBeDeleted.GId);
{
foreach (OpsJob jobToBeDeleted in jobsForObject)
{
await RemoveJobAndLogsAsync(ct, jobToBeDeleted.GId);
}
// Commit transaction if all commands succeed, transaction will auto-rollback
// when disposed if either commands fails
await transaction.CommitAsync();
}
catch (Exception ex)
{
throw ex;
}
} }
} }
/// <summary>
/// Remove job and logs for that job
/// </summary>
/// <param name="jobId"></param>
/// <param name="ct"></param>
internal static async Task DeleteJobAndLogAsync(Guid jobId, AyContext ct)
{
using (var transaction = await ct.Database.BeginTransactionAsync())
{
try
{
await RemoveJobAndLogsAsync(ct, jobId);
// Commit transaction if all commands succeed, transaction will auto-rollback
// when disposed if either commands fails
await transaction.CommitAsync();
}
catch (Exception ex)
{
throw ex;
}
}
}
/// <summary> /// <summary>
/// REmove the job and it's logs /// REmove the job and it's logs
/// </summary> /// </summary>
/// <param name="ct"></param>
/// <param name="jobIdToBeDeleted"></param> /// <param name="jobIdToBeDeleted"></param>
private static async Task RemoveJobAndLogsAsync(AyContext ct, Guid jobIdToBeDeleted) internal static async Task RemoveJobAndLogsAsync(Guid jobIdToBeDeleted)
{ {
// //delete logs var ct = ServiceProviderProvider.DBContext;
// await ct.Database.ExecuteSqlCommandAsync("delete from aopsjoblog where jobid = {0}", new object[] { jobIdToBeDeleted }); using (var transaction = await ct.Database.BeginTransactionAsync())
{
// //delete the job try
// await ct.Database.ExecuteSqlCommandAsync("delete from aopsjob where gid = {0}", new object[] { jobIdToBeDeleted }); {
//delete logs
//delete logs await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjoblog where jobid = {jobIdToBeDeleted}");
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjoblog where jobid = {jobIdToBeDeleted}"); //delete the job
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjob where gid = {jobIdToBeDeleted}");
//delete the job // Commit transaction if all commands succeed, transaction will auto-rollback
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjob where gid = {jobIdToBeDeleted}"); // when disposed if either commands fails
await transaction.CommitAsync();
}
catch (Exception ex)
{
throw ex;
}
}
} }
/// <summary> /// <summary>
/// Make a log entry for a job /// Make a log entry for a job
/// (no context version)
/// </summary> /// </summary>
/// <param name="jobId">(NOTE: Guid.empty indicates internal job)</param> /// <param name="jobId">(NOTE: Guid.empty indicates internal job)</param>
/// <param name="statusText"></param> /// <param name="statusText"></param>
///
internal static async Task<OpsJobLog> LogJobAsync(Guid jobId, string statusText) internal static async Task<OpsJobLog> LogJobAsync(Guid jobId, string statusText)
{ {
return await LogJobAsync(jobId, statusText, null); var ct = ServiceProviderProvider.DBContext;
}
/// <summary>
/// Make a log entry for a job
/// </summary>
/// <param name="jobId">(NOTE: Guid.empty indicates internal job)</param>
/// <param name="statusText"></param>
/// <param name="ct"></param>
internal static async Task<OpsJobLog> LogJobAsync(Guid jobId, string statusText, AyContext ct)
{
if (ct == null)
ct = ServiceProviderProvider.DBContext;
if (string.IsNullOrWhiteSpace(statusText)) if (string.IsNullOrWhiteSpace(statusText))
statusText = "No status provided"; statusText = "No status provided";
OpsJobLog newObj = new OpsJobLog(); OpsJobLog newObj = new OpsJobLog();
@@ -271,9 +226,9 @@ namespace AyaNova.Biz
/// </summary> /// </summary>
/// <param name="jobId"></param> /// <param name="jobId"></param>
/// <param name="newStatus"></param> /// <param name="newStatus"></param>
/// <param name="ct"></param> internal static async Task<OpsJob> UpdateJobStatusAsync(Guid jobId, JobStatus newStatus)
internal static async Task<OpsJob> UpdateJobStatusAsync(Guid jobId, JobStatus newStatus, AyContext ct)
{ {
var ct = ServiceProviderProvider.DBContext;
var oFromDb = await ct.OpsJob.SingleOrDefaultAsync(z => z.GId == jobId); var oFromDb = await ct.OpsJob.SingleOrDefaultAsync(z => z.GId == jobId);
if (oFromDb == null) return null; if (oFromDb == null) return null;
oFromDb.JobStatus = newStatus; oFromDb.JobStatus = newStatus;
@@ -289,37 +244,70 @@ namespace AyaNova.Biz
/// Process all jobs (stock jobs and those found in operations table) /// Process all jobs (stock jobs and those found in operations table)
/// </summary> /// </summary>
/// <returns></returns> /// <returns></returns>
internal static async Task ProcessJobsAsync(AyContext ct, AyaNova.Api.ControllerHelpers.ApiServerState serverState) internal static async Task ProcessJobsAsync(CancellationToken ctoken)
{ {
if (ActivelyProcessing) if (ActivelyProcessing)
{ {
System.Diagnostics.Debug.WriteLine("ProcessJobs called but actively processing other jobs so returning");
log.LogTrace("ProcessJobs called but actively processing other jobs so returning"); log.LogTrace("ProcessJobs called but actively processing other jobs so returning");
return; return;
} }
ActivelyProcessing = true; ActivelyProcessing = true;
try try
{ {
ctoken.ThrowIfCancellationRequested();
//Sweep jobs table
System.Diagnostics.Debug.WriteLine($"JobsBiz processing sweeper");
await CoreJobSweeper.DoSweepAsync(ctoken);//run exclusively
//BIZOBJECT DYNAMIC JOBS //BIZOBJECT DYNAMIC JOBS
//get a list of exclusive jobs that are due to happen //get a list of exclusive jobs that are due to happen
//Call into each item in turn //Call into each item in turn
List<OpsJob> exclusiveJobs = await GetReadyJobsExclusiveOnlyAsync(ct); List<OpsJob> exclusiveJobs = await GetReadyJobsExclusiveOnlyAsync();
foreach (OpsJob j in exclusiveJobs) foreach (OpsJob j in exclusiveJobs)
{ {
try try
{ {
await ProcessJobAsync(j, ct); ctoken.ThrowIfCancellationRequested();
System.Diagnostics.Debug.WriteLine($"JobsBiz processing exclusive biz job {j.Name}");
await ProcessJobAsync(j);
} }
catch (Exception ex) catch (Exception ex)
{ {
log.LogError(ex, $"ProcessJobs::Exclusive -> job {j.Name} failed with exception"); log.LogError(ex, $"ProcessJobs::Exclusive -> job {j.Name} failed with exception");
await LogJobAsync(j.GId, "Job failed with errors:", ct); await LogJobAsync(j.GId, "Job failed with errors:");
await LogJobAsync(j.GId, ExceptionUtil.ExtractAllExceptionMessages(ex), ct); await LogJobAsync(j.GId, ExceptionUtil.ExtractAllExceptionMessages(ex));
await UpdateJobStatusAsync(j.GId, JobStatus.Failed, ct); await UpdateJobStatusAsync(j.GId, JobStatus.Failed);
} }
} }
//Get a list of non-exlusive jobs that are due
ctoken.ThrowIfCancellationRequested();
System.Diagnostics.Debug.WriteLine($"JobsBiz processing exclusive license check");
//License check
long CurrentActiveCount = await UserBiz.ActiveCountAsync();
long LicensedUserCount = AyaNova.Core.License.ActiveKey.ActiveNumber;
// log.LogInformation("JobsBiz::Checking license active count");
if (CurrentActiveCount > LicensedUserCount)
{
var msg = $"E1020 - Active count exceeded capacity";
ServiceProviderProvider.ServerState.SetSystemLock(msg);
log.LogCritical(msg);
return;
}
ctoken.ThrowIfCancellationRequested();
//backup
System.Diagnostics.Debug.WriteLine($"JobsBiz processing backup");
await CoreJobBackup.DoWorkAsync();//sb exclusive
System.Diagnostics.Debug.WriteLine($"JobsBiz processing metrics snapshotter");
///////////////////////////////////////
//NON-EXCLUSIVE JOBS
//
//LOOKAT: Parallelize / background this block //LOOKAT: Parallelize / background this block
//http://www.dotnetcurry.com/dotnet/1360/concurrent-programming-dotnet-core //http://www.dotnetcurry.com/dotnet/1360/concurrent-programming-dotnet-core
@@ -329,42 +317,31 @@ namespace AyaNova.Biz
//This area may turn out to need a re-write in future, but I think it might only involve this block and ProcessJobAsync //This area may turn out to need a re-write in future, but I think it might only involve this block and ProcessJobAsync
//the actual individual objects that are responsible for jobs will likely not need a signature rewrite or anything (I hope) //the actual individual objects that are responsible for jobs will likely not need a signature rewrite or anything (I hope)
//For now I'z hoping that no job will be so slow that it can hold up all the other jobs indefinitely. //For now I'z hoping that no job will be so slow that it can hold up all the other jobs indefinitely.
ctoken.ThrowIfCancellationRequested();
List<OpsJob> sharedJobs = await GetReadyJobsNotExlusiveOnlyAsync(ct); List<OpsJob> sharedJobs = await GetReadyJobsNotExlusiveOnlyAsync();
foreach (OpsJob j in sharedJobs) foreach (OpsJob j in sharedJobs)
{ {
try try
{ {
await ProcessJobAsync(j, ct); ctoken.ThrowIfCancellationRequested();
System.Diagnostics.Debug.WriteLine($"JobsBiz processing NON-exclusive biz job {j.Name}");
// Task.Run(() => FireAway());
Task.Run(() => ProcessJobAsync(j), ctoken);
// await ProcessJobAsync(j, ct);
} }
catch (Exception ex) catch (Exception ex)
{ {
log.LogError(ex, $"ProcessJobs::Shared -> job {j.Name} failed with exception"); log.LogError(ex, $"ProcessJobs::Shared -> job {j.Name} failed with exception");
await LogJobAsync(j.GId, "Job failed with errors:", ct); await LogJobAsync(j.GId, "Job failed with errors:");
await LogJobAsync(j.GId, ExceptionUtil.ExtractAllExceptionMessages(ex), ct); await LogJobAsync(j.GId, ExceptionUtil.ExtractAllExceptionMessages(ex));
await UpdateJobStatusAsync(j.GId, JobStatus.Failed, ct); await UpdateJobStatusAsync(j.GId, JobStatus.Failed);
} }
} }
//STOCK JOBS //STOCK JOBS
//Sweep jobs table
await CoreJobSweeper.DoSweepAsync(ct);
//License check
long CurrentActiveCount = await UserBiz.ActiveCountAsync();
long LicensedUserCount = AyaNova.Core.License.ActiveKey.ActiveNumber;
// log.LogInformation("JobsBiz::Checking license active count");
if (CurrentActiveCount > LicensedUserCount)
{
var msg = $"E1020 - Active count exceeded capacity";
serverState.SetSystemLock(msg);
log.LogCritical(msg);
return;
}
//backup
await CoreJobBackup.DoWorkAsync(ct);
await CoreJobMetricsSnapshot.DoJobAsync(ct);
//Notifications //Notifications
@@ -377,6 +354,7 @@ namespace AyaNova.Biz
finally finally
{ {
ActivelyProcessing = false; ActivelyProcessing = false;
System.Diagnostics.Debug.WriteLine($"JobsBiz in Finally - completed run");
} }
} }
@@ -385,14 +363,13 @@ namespace AyaNova.Biz
/// Process a job by calling into it's biz object /// Process a job by calling into it's biz object
/// </summary> /// </summary>
/// <param name="job"></param> /// <param name="job"></param>
/// <param name="ct"></param>
/// <returns></returns> /// <returns></returns>
internal static async Task ProcessJobAsync(OpsJob job, AyContext ct) internal static async Task ProcessJobAsync(OpsJob job)
{ {
var JobDescription = $"{job.Name} {job.JobType.ToString()}"; var JobDescription = $"{job.Name} {job.JobType.ToString()}";
if (job.SubType != JobSubType.NotSet) if (job.SubType != JobSubType.NotSet)
JobDescription += $":{job.SubType}"; JobDescription += $":{job.SubType}";
await LogJobAsync(job.GId, $"Process job \"{JobDescription}\"", ct); await LogJobAsync(job.GId, $"Process job \"{JobDescription}\"");
log.LogDebug($"ProcessJobAsync -> Processing job {JobDescription}"); log.LogDebug($"ProcessJobAsync -> Processing job {JobDescription}");
IJobObject o = null; IJobObject o = null;
@@ -400,19 +377,19 @@ namespace AyaNova.Biz
{ {
case JobType.Backup: case JobType.Backup:
//This is called when on demand only, normal backups are processed above with normal system jobs //This is called when on demand only, normal backups are processed above with normal system jobs
await CoreJobBackup.DoWorkAsync(ct, true); await CoreJobBackup.DoWorkAsync(true);
await UpdateJobStatusAsync(job.GId, JobStatus.Completed, ct); await UpdateJobStatusAsync(job.GId, JobStatus.Completed);
break; break;
case JobType.TestJob: case JobType.TestJob:
o = (IJobObject)BizObjectFactory.GetBizObject(AyaType.ServerJob, ct); o = (IJobObject)BizObjectFactory.GetBizObject(AyaType.ServerJob);
break; break;
case JobType.SeedTestData: case JobType.SeedTestData:
o = (IJobObject)BizObjectFactory.GetBizObject(AyaType.TrialSeeder, ct); o = (IJobObject)BizObjectFactory.GetBizObject(AyaType.TrialSeeder);
break; break;
case JobType.BulkCoreBizObjectOperation: case JobType.BulkCoreBizObjectOperation:
//bulk op, hand off to biz object to deal with //bulk op, hand off to biz object to deal with
//note, convention is that there is an idList in job.jobinfo json if preselected else it's all objects of type //note, convention is that there is an idList in job.jobinfo json if preselected else it's all objects of type
o = (IJobObject)BizObjectFactory.GetBizObject(job.ObjectType, ct); o = (IJobObject)BizObjectFactory.GetBizObject(job.ObjectType);
break; break;
default: default:
throw new System.NotSupportedException($"ProcessJobAsync type {job.JobType.ToString()} is not supported"); throw new System.NotSupportedException($"ProcessJobAsync type {job.JobType.ToString()} is not supported");

5
server/AyaNova/biz/Search.cs
View File

@@ -246,8 +246,7 @@ namespace AyaNova.Biz
#endregion dosearch #endregion dosearch
#region Get info (excerpt) #region Get info (excerpt)
public static async Task<string> GetInfoAsync(AyContext ct, long translationId, public static async Task<string> GetInfoAsync(long translationId, AuthorizationRoles currentUserRoles, long userId, string phrase, int max, AyaType ayaType, long id)
AuthorizationRoles currentUserRoles, long userId, string phrase, int max, AyaType ayaType, long id)
{ {
//escape literal percentage signs first just in case they are searching for 50% off or something //escape literal percentage signs first just in case they are searching for 50% off or something
//https://www.postgresql.org/docs/current/functions-matching.html#FUNCTIONS-LIKE //https://www.postgresql.org/docs/current/functions-matching.html#FUNCTIONS-LIKE
@@ -262,7 +261,7 @@ namespace AyaNova.Biz
PhraseItems.ToArray(); PhraseItems.ToArray();
//get text //get text
ISearchAbleObject o = (ISearchAbleObject)BizObjectFactory.GetBizObject(ayaType, ct, userId, currentUserRoles); ISearchAbleObject o = (ISearchAbleObject)BizObjectFactory.GetBizObject(ayaType, userId, currentUserRoles);
//get extract //get extract
var searchParams = await o.GetSearchResultSummary(id); var searchParams = await o.GetSearchResultSummary(id);

14
server/AyaNova/biz/WorkOrderBiz.cs
View File

@@ -2441,8 +2441,8 @@ namespace AyaNova.Biz
private async Task ProcessBulkJobAsync(OpsJob job) private async Task ProcessBulkJobAsync(OpsJob job)
{ {
await JobsBiz.UpdateJobStatusAsync(job.GId, JobStatus.Running, ct); await JobsBiz.UpdateJobStatusAsync(job.GId, JobStatus.Running);
await JobsBiz.LogJobAsync(job.GId, $"Bulk job {job.SubType} started...", ct); await JobsBiz.LogJobAsync(job.GId, $"Bulk job {job.SubType} started...");
List<long> idList = new List<long>(); List<long> idList = new List<long>();
long ProcessedObjectCount = 0; long ProcessedObjectCount = 0;
JObject jobData = JObject.Parse(job.JobInfo); JObject jobData = JObject.Parse(job.JobInfo);
@@ -2475,7 +2475,7 @@ namespace AyaNova.Biz
{ {
o = await PutWorkOrderGraphItem(job.ObjectType, o); o = await PutWorkOrderGraphItem(job.ObjectType, o);
if (o == null) if (o == null)
await JobsBiz.LogJobAsync(job.GId, $"Error processing item {id}: {GetErrorsAsString()}", ct); await JobsBiz.LogJobAsync(job.GId, $"Error processing item {id}: {GetErrorsAsString()}");
else else
ProcessedObjectCount++; ProcessedObjectCount++;
} }
@@ -2484,12 +2484,12 @@ namespace AyaNova.Biz
} }
catch (Exception ex) catch (Exception ex)
{ {
await JobsBiz.LogJobAsync(job.GId, $"Error processing item {id}", ct); await JobsBiz.LogJobAsync(job.GId, $"Error processing item {id}");
await JobsBiz.LogJobAsync(job.GId, ExceptionUtil.ExtractAllExceptionMessages(ex), ct); await JobsBiz.LogJobAsync(job.GId, ExceptionUtil.ExtractAllExceptionMessages(ex));
} }
} }
await JobsBiz.LogJobAsync(job.GId, $"Bulk job {job.SubType} processed {ProcessedObjectCount} of {idList.Count}", ct); await JobsBiz.LogJobAsync(job.GId, $"Bulk job {job.SubType} processed {ProcessedObjectCount} of {idList.Count}");
await JobsBiz.UpdateJobStatusAsync(job.GId, JobStatus.Completed, ct); await JobsBiz.UpdateJobStatusAsync(job.GId, JobStatus.Completed);
} }

33
server/AyaNova/generator/CoreJobMetricsSnapshot.cs
View File

@@ -27,17 +27,18 @@ namespace AyaNova.Biz
#if(DEBUG) #if(DEBUG)
private static TimeSpan tsOneMinute = new TimeSpan(0, 0, 10); private static TimeSpan tsOneMinute = new TimeSpan(0, 0, 10);
#else #else
private static TimeSpan tsOneMinute = new TimeSpan(0, 1, 0); private static TimeSpan tsOneMinute = new TimeSpan(0, 1, 0);
#endif #endif
private static TimeSpan tsOneHour = new TimeSpan(1, 0, 0); private static TimeSpan tsOneHour = new TimeSpan(1, 0, 0);
private static TimeSpan ts24Hours = new TimeSpan(24, 0, 0); private static TimeSpan ts24Hours = new TimeSpan(24, 0, 0);
//////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////
// DoAsync // DoAsync
// //
public static async Task DoJobAsync(AyContext ct) public static void DoJob()
{ {
#region worksheet
//https://github.com/sebastienros/memoryleak/blob/master/src/MemoryLeak/MemoryLeak/Controllers/DiagnosticsController.cs //https://github.com/sebastienros/memoryleak/blob/master/src/MemoryLeak/MemoryLeak/Controllers/DiagnosticsController.cs
// //DATA TYPES .net to postgres map // //DATA TYPES .net to postgres map
//http://www.npgsql.org/doc/types/basic.html //http://www.npgsql.org/doc/types/basic.html
@@ -153,6 +154,7 @@ from generate_series(1, 525600) s(i)
,...); ,...);
*/ */
#endregion worksheet
//Nothing is gathered less than one minute frequency //Nothing is gathered less than one minute frequency
if (!DateUtil.IsAfterDuration(_lastSnapshot, tsOneMinute)) if (!DateUtil.IsAfterDuration(_lastSnapshot, tsOneMinute))
@@ -196,12 +198,15 @@ from generate_series(1, 525600) s(i)
var Gen2 = GC.CollectionCount(2);//integer var Gen2 = GC.CollectionCount(2);//integer
var CPU = _cpu;// double precision var CPU = _cpu;// double precision
System.Diagnostics.Debug.WriteLine($"MM Snapshot, cpu: {CPU}");
//write to db using (AyContext ct = ServiceProviderProvider.DBContext)
MetricMM mm = new MetricMM(Allocated, WorkingSet, PrivateBytes, Gen0, Gen1, Gen2, CPU); {
await ct.MetricMM.AddAsync(mm); //write to db
await ct.SaveChangesAsync(); MetricMM mm = new MetricMM(Allocated, WorkingSet, PrivateBytes, Gen0, Gen1, Gen2, CPU);
ct.MetricMM.Add(mm);
ct.SaveChanges();
System.Diagnostics.Debug.WriteLine("MM SAVED");
}
///////////////////////////////////////////// /////////////////////////////////////////////
@@ -270,17 +275,17 @@ from generate_series(1, 525600) s(i)
// //
DateTime ClearDate = DateTime.UtcNow - tsDataRetention; DateTime ClearDate = DateTime.UtcNow - tsDataRetention;
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from ametricmm where t > {ClearDate}"); using (AyContext ct = ServiceProviderProvider.DBContext)
{
ct.Database.ExecuteSqlInterpolated($"delete from ametricmm where t < {ClearDate}");
}
} }
_lastSnapshot = now; _lastSnapshot = now;
{
DateTime ClearDate = DateTime.UtcNow - tsDataRetention;
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from ametricmm where t > {ClearDate}");
}
} }

61
server/AyaNova/generator/CoreJobSweeper.cs
View File

@@ -4,7 +4,7 @@ using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using AyaNova.Models; using AyaNova.Models;
using System.Threading;
namespace AyaNova.Biz namespace AyaNova.Biz
{ {
@@ -28,44 +28,49 @@ namespace AyaNova.Biz
//////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////////
// DoSweep // DoSweep
// //
public static async Task DoSweepAsync(AyContext ct) public static async Task DoSweepAsync(CancellationToken ctoken)
{ {
ctoken.ThrowIfCancellationRequested();
//This will get triggered roughly every minute, but we don't want to sweep that frequently //This will get triggered roughly every minute, but we don't want to sweep that frequently
if (DateTime.UtcNow - lastSweep < SWEEP_EVERY_INTERVAL) if (DateTime.UtcNow - lastSweep < SWEEP_EVERY_INTERVAL)
return; return;
log.LogTrace("Sweep starting"); log.LogTrace("Sweep starting");
using (AyContext ct = AyaNova.Util.ServiceProviderProvider.DBContext)
{
//SWEEP SUCCESSFUL JOBS //SWEEP SUCCESSFUL JOBS
//calculate cutoff to delete //calculate cutoff to delete
DateTime dtDeleteCutoff = DateTime.UtcNow - SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN; DateTime dtDeleteCutoff = DateTime.UtcNow - SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Completed); await sweepAsync(ct, dtDeleteCutoff, JobStatus.Completed, ctoken);
//SWEEP FAILED JOBS ctoken.ThrowIfCancellationRequested();
//calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Failed);
//KILL STUCK JOBS
//calculate cutoff to delete
DateTime dtRunningDeadline = DateTime.UtcNow - RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN;
await killStuckJobsAsync(ct, dtRunningDeadline);
//SWEEP INTERNAL JOB LOG
//calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - INTERNAL_JOBS_LOGS_DELETE_AFTER_THIS_TIMESPAN;
await SweepInternalJobsLogsAsync(ct, dtDeleteCutoff);
//SWEEP FAILED JOBS
//calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Failed, ctoken);
ctoken.ThrowIfCancellationRequested();
//KILL STUCK JOBS
//calculate cutoff to delete
DateTime dtRunningDeadline = DateTime.UtcNow - RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN;
await killStuckJobsAsync(ct, dtRunningDeadline);
ctoken.ThrowIfCancellationRequested();
//SWEEP INTERNAL JOB LOG
//calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - INTERNAL_JOBS_LOGS_DELETE_AFTER_THIS_TIMESPAN;
await SweepInternalJobsLogsAsync(ct, dtDeleteCutoff);
ctoken.ThrowIfCancellationRequested();
}
lastSweep = DateTime.UtcNow; lastSweep = DateTime.UtcNow;
} }
private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus) private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus, CancellationToken ctoken)//AyContext ct,
{ {
// AyContext ct = ServiceProviderProvider.DBContext;
//Get the deleteable succeeded jobs list //Get the deleteable succeeded jobs list
var jobs = await ct.OpsJob var jobs = await ct.OpsJob
.AsNoTracking() .AsNoTracking()
@@ -79,11 +84,12 @@ namespace AyaNova.Biz
{ {
try try
{ {
await JobsBiz.DeleteJobAndLogAsync(j.GId, ct); ctoken.ThrowIfCancellationRequested();
await JobsBiz.RemoveJobAndLogsAsync(j.GId);
} }
catch (Exception ex) catch (Exception ex)
{ {
log.LogError(ex, "sweepAsync exception calling JobsBiz.DeleteJobAndLogAsync"); log.LogError(ex, "sweepAsync exception calling JobsBiz.RemoveJobAndLogsAsync");
//for now just throw it but this needs to be removed when logging added and better handling //for now just throw it but this needs to be removed when logging added and better handling
throw (ex); throw (ex);
} }
@@ -94,11 +100,12 @@ namespace AyaNova.Biz
/// <summary> /// <summary>
/// Kill jobs that have been stuck in "running" state for too long /// Kill jobs that have been stuck in "running" state for too long
/// </summary> /// </summary>
/// <param name="ct"></param>
/// <param name="dtRunningDeadline"></param> /// <param name="dtRunningDeadline"></param>
/// <returns></returns> /// <returns></returns>
private static async Task killStuckJobsAsync(AyContext ct, DateTime dtRunningDeadline) private static async Task killStuckJobsAsync(AyContext ct, DateTime dtRunningDeadline)
{ {
// AyContext ct = ServiceProviderProvider.DBContext;
//Get the deleteable succeeded jobs list //Get the deleteable succeeded jobs list
var jobs = await ct.OpsJob var jobs = await ct.OpsJob
.AsNoTracking() .AsNoTracking()
@@ -120,7 +127,7 @@ namespace AyaNova.Biz
private static async Task SweepInternalJobsLogsAsync(AyContext ct, DateTime dtDeleteCutoff) private static async Task SweepInternalJobsLogsAsync(AyContext ct, DateTime dtDeleteCutoff)
{ {
// AyContext ct = ServiceProviderProvider.DBContext;
//Get the deleteable list (this is for reporting, could easily just do it in one go) //Get the deleteable list (this is for reporting, could easily just do it in one go)
var logs = await ct.OpsJobLog var logs = await ct.OpsJobLog
.AsNoTracking() .AsNoTracking()

51
server/AyaNova/generator/Generate.cs
View File

@@ -6,6 +6,7 @@ using Microsoft.Extensions.DependencyInjection;
using AyaNova.Models; using AyaNova.Models;
using AyaNova.Api.ControllerHelpers; using AyaNova.Api.ControllerHelpers;
using AyaNova.Biz; using AyaNova.Biz;
using AyaNova.Util;
namespace AyaNova.Generator namespace AyaNova.Generator
{ {
@@ -27,6 +28,7 @@ namespace AyaNova.Generator
// private readonly ApiServerState serverState; // private readonly ApiServerState serverState;
private readonly IServiceProvider provider; private readonly IServiceProvider provider;
private const int MAXIMUM_TIME_ALLOWED_FOR_PROCESSING_ALL_JOBS = 1 * 60 * 1000;//1 minutes TEST TEST TEST #####
#if(DEBUG) #if(DEBUG)
private const int GENERATE_SECONDS = 5; private const int GENERATE_SECONDS = 5;
@@ -53,7 +55,11 @@ namespace AyaNova.Generator
/*
todo: improve this
it should timeout: https://stackoverflow.com/questions/23476576/cancellationtoken-timeout-vs-task-delay-and-timeout
it should never stop running no matter what unless teh server shuts down
*/
protected override async Task ExecuteAsync(CancellationToken stoppingToken) protected override async Task ExecuteAsync(CancellationToken stoppingToken)
@@ -73,32 +79,37 @@ namespace AyaNova.Generator
{ {
log.LogDebug($"GeneratorService task doing background work."); log.LogDebug($"GeneratorService task doing background work.");
using (IServiceScope scope = provider.CreateScope()) ApiServerState serverState = ServiceProviderProvider.ServerState;
//=================================================================
try
{ {
AyContext ct = scope.ServiceProvider.GetRequiredService<AyContext>(); if (!serverState.IsOpen)
ApiServerState serverState = scope.ServiceProvider.GetRequiredService<ApiServerState>();
//=================================================================
try
{ {
if (!serverState.IsOpen) log.LogDebug($"GeneratorService: ServerState is closed returning without processing jobs, will try again next iteration");
{
log.LogDebug($"GeneratorService: ServerState is closed returning without processing jobs, will try again next iteration");
}
else
{
await JobsBiz.ProcessJobsAsync(ct, serverState);
}
} }
catch (Exception ex) else
{ {
log.LogError(ex, "Generate::ProcessJobs result in exception error "); System.Diagnostics.Debug.WriteLine($"### GENERATE calling JobsBiz.ProcessJobs");
//Before anything capture metrics
// await CoreJobMetricsSnapshot.DoJobAsync(ct);
//Task.Run(() => CoreJobMetricsSnapshot.DoJob());
//deliberately calling this non-async
//it needs to be fast and efficient
CoreJobMetricsSnapshot.DoJob();
//TODO: this should be big timeout and then inside the process jobs each job has it's own timeout
await TaskUtil.WithTimeoutAfterStart(ctoken => JobsBiz.ProcessJobsAsync(ctoken), TimeSpan.FromMilliseconds(MAXIMUM_TIME_ALLOWED_FOR_PROCESSING_ALL_JOBS));
// await JobsBiz.ProcessJobsAsync(ct, serverState);
System.Diagnostics.Debug.WriteLine($"### GENERATE BACK FROM calling JobsBiz.ProcessJobs");
} }
//=================================================================
} }
catch (Exception ex)
{
log.LogError(ex, "Generate::ProcessJobs result in exception error ");
}
//=================================================================
} }
await Task.Delay((GENERATE_SECONDS * 1000), stoppingToken); await Task.Delay((GENERATE_SECONDS * 1000), stoppingToken);
justStarted = false; justStarted = false;

8
server/AyaNova/util/ServiceProviderProvider.cs
View File

@@ -58,6 +58,14 @@ namespace AyaNova.Util
} }
} }
internal static AyaNova.Api.ControllerHelpers.ApiServerState ServerState
{
get
{
return Scope.ServiceProvider.GetRequiredService<AyaNova.Api.ControllerHelpers.ApiServerState>();
}
}
// https://docs.microsoft.com/en-us/aspnet/core/fundamentals/http-requests?view=aspnetcore-3.1 // https://docs.microsoft.com/en-us/aspnet/core/fundamentals/http-requests?view=aspnetcore-3.1
internal static System.Net.Http.IHttpClientFactory HttpClientFactory internal static System.Net.Http.IHttpClientFactory HttpClientFactory
{ {