admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Browse Source
This commit is contained in:
John Cardinal
2021-12-23 15:15:14 +00:00
parent 7d357270f6
commit e754230cbd
9 changed files with 42 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
server/AyaNova/util/ReportProcessManager.cs
View File

@@ -6,20 +6,10 @@ using Microsoft.Extensions.Logging;
namespace AyaNova.Util
{
/// <summary>
/// Used by reporting system to ensure headless browsers don't hang around in an untimely manner chewing up resources
/// needed due to bugs in puppeteersharp where it won't close the browser on timeout properly
/// also zombie process issues in linux etc, this just ensures it's safe
/// This is triggered when a report is rendered on demand
/// in other words demand drives whether it kills long running renders or not
/// this is by design to allow a scenario where a super long running report can still be run off hours (for example)
/// </summary>
//Track processes and kill any that go past their expiry date
internal static class ReportRenderManager
{
/*
expired processes are removed by the act of trying to get a new slot so in this way it still supports running super long reports overnight for example as long as there is no contention
The other way was by a job that looks for expired processes but that would mean all old jobs would expire all the time so there would be an issue with huge reports never working
*/
//thread safe collection for unordered items, optimized for single thread produce/consume (which is the norm here) but supports multithread produce / consume (which is needed for separate cleanup job)
internal static ConcurrentBag<ReportRenderInstanceInfo> _baginstances;// = new ConcurrentBag<ReportRenderInstanceInfo>();
@@ -34,45 +24,35 @@ namespace AyaNova.Util
internal int ReporterProcessId { get; set; }
internal DateTime Expires { get; set; }
internal ReportRenderInstanceInfo(int processId)
internal ReportRenderInstanceInfo(int processId, DateTime expires)
{
ReporterProcessId = processId;
Expires = DateTime.UtcNow.AddMilliseconds(ServerBootConfig.AYANOVA_REPORT_RENDERING_TIMEOUT);
Expires = expires;
}
}
internal static bool RenderSlotAvailable(ILogger log)
internal static void KillExpiredRenders(ILogger log)
{
log.LogTrace("RenderSlotAvailable check");
//var count = _baginstances.Count;
// #if (DEBUG)
// log.LogInformation($"DBG: RenderSlotAvailable check, there are currently {_baginstances.Count} instances in the bag");
// #endif
if (_baginstances.Count >= ServerBootConfig.AYANOVA_REPORT_RENDERING_MAX_INSTANCES)
log.LogTrace("Clear expired render check");
//check for expired and remove
var Instances = _baginstances.ToArray();
var dtNow = DateTime.UtcNow;
foreach (ReportRenderInstanceInfo i in Instances)
{
log.LogTrace($"RenderSlotAvailable there are no free report rendering slots available, current count is {_baginstances.Count}, checking for expired slots to force closed");
//check for expired and remove
var Instances = _baginstances.ToArray();
var dtNow = DateTime.UtcNow;
foreach (ReportRenderInstanceInfo i in Instances)
if (i.Expires < dtNow)
{
if (i.Expires < dtNow)
{
// #if (DEBUG)
// log.LogInformation($"DBG: RenderSlotAvailable attempting kill of expired process {i.ReporterProcessId}");
// #endif
ForceCloseProcess(i, log);
}
#if (DEBUG)
log.LogInformation($"DBG: KillExpiredRenders attempting kill of expired process {i.ReporterProcessId}");
#endif
ForceCloseProcess(i, log);
}
}
//allow to continue if there are now fewer than max instances in the bag
return _baginstances.Count < ServerBootConfig.AYANOVA_REPORT_RENDERING_MAX_INSTANCES;
}
internal static bool ForceCloseProcess(ReportRenderInstanceInfo instance, ILogger log)
{
log.LogTrace($"ForceCloseProcess on instance id {instance.ReporterProcessId} started {instance.Expires.ToString()} utc");
log.LogTrace($"ForceCloseProcess on report render instance id {instance.ReporterProcessId} exired {instance.Expires.ToString()} utc");
try
{
var p = Process.GetProcessById(instance.ReporterProcessId);
@@ -83,8 +63,7 @@ namespace AyaNova.Util
p.Kill();
if (p.HasExited == false)
{
log.LogDebug($"RenderSlotAvailable oldest slot could not be stopped");
log.LogDebug($"Expired report render instance id {instance.ReporterProcessId} could not be force closed");
return false;//can't kill it so can't free up a slot
}
}
@@ -105,12 +84,12 @@ namespace AyaNova.Util
}
}
internal static void AddProcess(int processId, ILogger log)
internal static void AddProcess(int processId, DateTime expires, ILogger log)
{
// #if (DEBUG)
// log.LogInformation($"DBG: RenderSlotAvailable::AddProcess {processId} in the bag");
// #endif
_baginstances.Add(new ReportRenderInstanceInfo(processId));
_baginstances.Add(new ReportRenderInstanceInfo(processId, expires));
// #if (DEBUG)
// log.LogInformation($"DBG: RenderSlotAvailable::AddProcess, there are currently {_baginstances.Count} instances in the bag");