admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

much fuckery

Browse Source
This commit is contained in:
John Cardinal
2020-05-25 20:34:11 +00:00
parent d977728a22
commit e90dbe0f71
9 changed files with 218 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
server/AyaNova/generator/CoreJobMetricsSnapshot.cs
View File

@@ -27,17 +27,18 @@ namespace AyaNova.Biz
#if(DEBUG)
private static TimeSpan tsOneMinute = new TimeSpan(0, 0, 10);
#else
#else
private static TimeSpan tsOneMinute = new TimeSpan(0, 1, 0);
#endif
#endif
private static TimeSpan tsOneHour = new TimeSpan(1, 0, 0);
private static TimeSpan ts24Hours = new TimeSpan(24, 0, 0);
////////////////////////////////////////////////////////////////////////////////////////////////
// DoAsync
//
public static async Task DoJobAsync(AyContext ct)
public static void DoJob()
{
#region worksheet
//https://github.com/sebastienros/memoryleak/blob/master/src/MemoryLeak/MemoryLeak/Controllers/DiagnosticsController.cs
// //DATA TYPES .net to postgres map
//http://www.npgsql.org/doc/types/basic.html
@@ -153,6 +154,7 @@ from generate_series(1, 525600) s(i)
,...);
*/
#endregion worksheet
//Nothing is gathered less than one minute frequency
if (!DateUtil.IsAfterDuration(_lastSnapshot, tsOneMinute))
@@ -196,12 +198,15 @@ from generate_series(1, 525600) s(i)
var Gen2 = GC.CollectionCount(2);//integer
var CPU = _cpu;// double precision
//write to db
MetricMM mm = new MetricMM(Allocated, WorkingSet, PrivateBytes, Gen0, Gen1, Gen2, CPU);
await ct.MetricMM.AddAsync(mm);
await ct.SaveChangesAsync();
System.Diagnostics.Debug.WriteLine($"MM Snapshot, cpu: {CPU}");
using (AyContext ct = ServiceProviderProvider.DBContext)
{
//write to db
MetricMM mm = new MetricMM(Allocated, WorkingSet, PrivateBytes, Gen0, Gen1, Gen2, CPU);
ct.MetricMM.Add(mm);
ct.SaveChanges();
System.Diagnostics.Debug.WriteLine("MM SAVED");
}
/////////////////////////////////////////////
@@ -270,17 +275,17 @@ from generate_series(1, 525600) s(i)
//
DateTime ClearDate = DateTime.UtcNow - tsDataRetention;
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from ametricmm where t > {ClearDate}");
using (AyContext ct = ServiceProviderProvider.DBContext)
{
ct.Database.ExecuteSqlInterpolated($"delete from ametricmm where t < {ClearDate}");
}
}
_lastSnapshot = now;
{
DateTime ClearDate = DateTime.UtcNow - tsDataRetention;
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from ametricmm where t > {ClearDate}");
}
}

63
server/AyaNova/generator/CoreJobSweeper.cs
View File

@@ -4,7 +4,7 @@ using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using AyaNova.Models;
using System.Threading;
namespace AyaNova.Biz
{
@@ -28,44 +28,49 @@ namespace AyaNova.Biz
////////////////////////////////////////////////////////////////////////////////////////////////
// DoSweep
//
public static async Task DoSweepAsync(AyContext ct)
public static async Task DoSweepAsync(CancellationToken ctoken)
{
ctoken.ThrowIfCancellationRequested();
//This will get triggered roughly every minute, but we don't want to sweep that frequently
if (DateTime.UtcNow - lastSweep < SWEEP_EVERY_INTERVAL)
return;
log.LogTrace("Sweep starting");
using (AyContext ct = AyaNova.Util.ServiceProviderProvider.DBContext)
{
//SWEEP SUCCESSFUL JOBS
//calculate cutoff to delete
DateTime dtDeleteCutoff = DateTime.UtcNow - SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Completed);
//SWEEP SUCCESSFUL JOBS
//calculate cutoff to delete
DateTime dtDeleteCutoff = DateTime.UtcNow - SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Completed, ctoken);
//SWEEP FAILED JOBS
//calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Failed);
//KILL STUCK JOBS
//calculate cutoff to delete
DateTime dtRunningDeadline = DateTime.UtcNow - RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN;
await killStuckJobsAsync(ct, dtRunningDeadline);
//SWEEP INTERNAL JOB LOG
//calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - INTERNAL_JOBS_LOGS_DELETE_AFTER_THIS_TIMESPAN;
await SweepInternalJobsLogsAsync(ct, dtDeleteCutoff);
ctoken.ThrowIfCancellationRequested();
//SWEEP FAILED JOBS
//calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Failed, ctoken);
ctoken.ThrowIfCancellationRequested();
//KILL STUCK JOBS
//calculate cutoff to delete
DateTime dtRunningDeadline = DateTime.UtcNow - RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN;
await killStuckJobsAsync(ct, dtRunningDeadline);
ctoken.ThrowIfCancellationRequested();
//SWEEP INTERNAL JOB LOG
//calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - INTERNAL_JOBS_LOGS_DELETE_AFTER_THIS_TIMESPAN;
await SweepInternalJobsLogsAsync(ct, dtDeleteCutoff);
ctoken.ThrowIfCancellationRequested();
}
lastSweep = DateTime.UtcNow;
}
private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus)
private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus, CancellationToken ctoken)//AyContext ct,
{
// AyContext ct = ServiceProviderProvider.DBContext;
//Get the deleteable succeeded jobs list
var jobs = await ct.OpsJob
.AsNoTracking()
@@ -79,11 +84,12 @@ namespace AyaNova.Biz
{
try
{
await JobsBiz.DeleteJobAndLogAsync(j.GId, ct);
ctoken.ThrowIfCancellationRequested();
await JobsBiz.RemoveJobAndLogsAsync(j.GId);
}
catch (Exception ex)
{
log.LogError(ex, "sweepAsync exception calling JobsBiz.DeleteJobAndLogAsync");
log.LogError(ex, "sweepAsync exception calling JobsBiz.RemoveJobAndLogsAsync");
//for now just throw it but this needs to be removed when logging added and better handling
throw (ex);
}
@@ -93,12 +99,13 @@ namespace AyaNova.Biz
/// <summary>
/// Kill jobs that have been stuck in "running" state for too long
/// </summary>
/// <param name="ct"></param>
/// </summary>
/// <param name="dtRunningDeadline"></param>
/// <returns></returns>
private static async Task killStuckJobsAsync(AyContext ct, DateTime dtRunningDeadline)
{
// AyContext ct = ServiceProviderProvider.DBContext;
//Get the deleteable succeeded jobs list
var jobs = await ct.OpsJob
.AsNoTracking()
@@ -120,7 +127,7 @@ namespace AyaNova.Biz
private static async Task SweepInternalJobsLogsAsync(AyContext ct, DateTime dtDeleteCutoff)
{
// AyContext ct = ServiceProviderProvider.DBContext;
//Get the deleteable list (this is for reporting, could easily just do it in one go)
var logs = await ct.OpsJobLog
.AsNoTracking()

51
server/AyaNova/generator/Generate.cs
View File

@@ -6,6 +6,7 @@ using Microsoft.Extensions.DependencyInjection;
using AyaNova.Models;
using AyaNova.Api.ControllerHelpers;
using AyaNova.Biz;
using AyaNova.Util;
namespace AyaNova.Generator
{
@@ -27,6 +28,7 @@ namespace AyaNova.Generator
// private readonly ApiServerState serverState;
private readonly IServiceProvider provider;
private const int MAXIMUM_TIME_ALLOWED_FOR_PROCESSING_ALL_JOBS = 1 * 60 * 1000;//1 minutes TEST TEST TEST #####
#if(DEBUG)
private const int GENERATE_SECONDS = 5;
@@ -53,7 +55,11 @@ namespace AyaNova.Generator
/*
todo: improve this
it should timeout: https://stackoverflow.com/questions/23476576/cancellationtoken-timeout-vs-task-delay-and-timeout
it should never stop running no matter what unless teh server shuts down
*/
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
@@ -73,32 +79,37 @@ namespace AyaNova.Generator
{
log.LogDebug($"GeneratorService task doing background work.");
using (IServiceScope scope = provider.CreateScope())
ApiServerState serverState = ServiceProviderProvider.ServerState;
//=================================================================
try
{
AyContext ct = scope.ServiceProvider.GetRequiredService<AyContext>();
ApiServerState serverState = scope.ServiceProvider.GetRequiredService<ApiServerState>();
//=================================================================
try
if (!serverState.IsOpen)
{
if (!serverState.IsOpen)
{
log.LogDebug($"GeneratorService: ServerState is closed returning without processing jobs, will try again next iteration");
}
else
{
await JobsBiz.ProcessJobsAsync(ct, serverState);
}
log.LogDebug($"GeneratorService: ServerState is closed returning without processing jobs, will try again next iteration");
}
catch (Exception ex)
else
{
log.LogError(ex, "Generate::ProcessJobs result in exception error ");
System.Diagnostics.Debug.WriteLine($"### GENERATE calling JobsBiz.ProcessJobs");
//Before anything capture metrics
// await CoreJobMetricsSnapshot.DoJobAsync(ct);
//Task.Run(() => CoreJobMetricsSnapshot.DoJob());
//deliberately calling this non-async
//it needs to be fast and efficient
CoreJobMetricsSnapshot.DoJob();
//TODO: this should be big timeout and then inside the process jobs each job has it's own timeout
await TaskUtil.WithTimeoutAfterStart(ctoken => JobsBiz.ProcessJobsAsync(ctoken), TimeSpan.FromMilliseconds(MAXIMUM_TIME_ALLOWED_FOR_PROCESSING_ALL_JOBS));
// await JobsBiz.ProcessJobsAsync(ct, serverState);
System.Diagnostics.Debug.WriteLine($"### GENERATE BACK FROM calling JobsBiz.ProcessJobs");
}
//=================================================================
}
catch (Exception ex)
{
log.LogError(ex, "Generate::ProcessJobs result in exception error ");
}
//=================================================================
}
await Task.Delay((GENERATE_SECONDS * 1000), stoppingToken);
justStarted = false;