121 lines
5.4 KiB
C#
121 lines
5.4 KiB
C#
using System;
|
|
using System.Collections.Concurrent;
|
|
using System.Diagnostics;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace AyaNova.Util
|
|
{
|
|
|
|
/// <summary>
|
|
/// Used by reporting system to ensure headless browsers don't hang around in an untimely manner chewing up resources
|
|
/// needed due to bugs in puppeteersharp where it won't close the browser on timeout properly
|
|
/// also zombie process issues in linux etc, this just ensures it's safe
|
|
/// This is triggered when a report is rendered on demand
|
|
/// in other words demand drives whether it kills long running renders or not
|
|
/// this is by design to allow a scenario where a super long running report can still be run off hours (for example)
|
|
/// </summary>
|
|
internal static class ReportRenderManager
|
|
{
|
|
/*
|
|
expired processes are removed by the act of tryign to get a new slot so in this way it still supports running super long reports overnight for example as long as there is no contention
|
|
The other way was by a job that looks for expired processes but that would mean all old jobs would expire all the time so there would be an issue with huge reports never working
|
|
*/
|
|
|
|
//thread safe collection for unordered items, optimized for single thread produce/consume (which is the norm here) but supports multithread produce / consume (which is needed for separate cleanup job)
|
|
private static ConcurrentBag<ReportRenderInstanceInfo> _baginstances = new ConcurrentBag<ReportRenderInstanceInfo>();
|
|
|
|
public class ReportRenderInstanceInfo
|
|
{
|
|
public int ReporterProcessId { get; set; }
|
|
public DateTime Expires { get; set; }
|
|
|
|
public ReportRenderInstanceInfo(int processId)
|
|
{
|
|
ReporterProcessId = processId;
|
|
Expires = DateTime.UtcNow.AddMilliseconds(ServerBootConfig.AYANOVA_REPORT_RENDERING_TIMEOUT);
|
|
}
|
|
}
|
|
|
|
|
|
public static bool RenderSlotAvailable(ILogger log)
|
|
{
|
|
log.LogTrace("RenderSlotAvailable check");
|
|
var count = _baginstances.Count;
|
|
// #if (DEBUG)
|
|
// log.LogInformation($"DBG: RenderSlotAvailable check, there are currently {count} instances in the bag");
|
|
// #endif
|
|
if (count >= ServerBootConfig.AYANOVA_REPORT_RENDERING_MAX_INSTANCES)
|
|
{
|
|
log.LogTrace($"RenderSlotAvailable there are no free report rendering slots available, current count is {count}, checking for expired slots to force closed");
|
|
//check for expired and remove
|
|
var Instances = _baginstances.ToArray();
|
|
var dtNow = DateTime.UtcNow;
|
|
foreach (ReportRenderInstanceInfo i in Instances)
|
|
{
|
|
if (i.Expires < dtNow)
|
|
{
|
|
// #if (DEBUG)
|
|
// log.LogInformation($"DBG: RenderSlotAvailable attempting kill of expired process {i.ReporterProcessId}");
|
|
// #endif
|
|
ForceCloseProcess(i, log);
|
|
}
|
|
}
|
|
}
|
|
//allow to continue if there are now fewer than max instances in the bag
|
|
return _baginstances.Count < ServerBootConfig.AYANOVA_REPORT_RENDERING_MAX_INSTANCES;
|
|
}
|
|
|
|
private static bool ForceCloseProcess(ReportRenderInstanceInfo instance, ILogger log)
|
|
{
|
|
log.LogTrace($"ForceCloseProcess on instance id {instance.ReporterProcessId} started {instance.Expires.ToString()} utc");
|
|
try
|
|
{
|
|
var p = Process.GetProcessById(instance.ReporterProcessId);
|
|
if (p != null)
|
|
{
|
|
//we have an existing process
|
|
//try to kill it
|
|
p.Kill();
|
|
if (p.HasExited == false)
|
|
{
|
|
|
|
log.LogDebug($"RenderSlotAvailable oldest slot could not be stopped");
|
|
return false;//can't kill it so can't free up a slot
|
|
}
|
|
}
|
|
//remove it from the list, it's either gone or killed at this point
|
|
//this would not be unexpected since it will normally just close on it's own
|
|
//at the finally in render report
|
|
_baginstances.TryTake(out instance);
|
|
return true;//process that was there is now not there so while not perfect system we will consider it free
|
|
|
|
}
|
|
catch (ArgumentException)
|
|
{
|
|
//do nothing, this is normal, the process could not be found and this means it's already been removed:
|
|
//ArgumentException
|
|
//The process specified by the processId parameter is not running. The identifier might be expired.
|
|
_baginstances.TryTake(out instance);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
internal static void AddProcess(int processId)
|
|
{
|
|
_baginstances.Add(new ReportRenderInstanceInfo(processId));
|
|
}
|
|
|
|
internal static void RemoveProcess(int processId, ILogger log)
|
|
{
|
|
foreach (var i in _baginstances)
|
|
{
|
|
if (i.ReporterProcessId == processId)
|
|
{
|
|
ForceCloseProcess(i, log);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
}//eoc
|
|
}//eons |