From bc802e29568ec8303710677312ee71c82a397307 Mon Sep 17 00:00:00 2001 From: John Cardinal Date: Wed, 27 Oct 2021 21:22:00 +0000 Subject: [PATCH] Report render process manager v2.0 --- .vscode/launch.json | 1 + .../docs/ops-config-environment-variables.md | 1 + .../ops-config-report-rendering-timeout.md | 5 +- .../AyaNova/Controllers/ReportController.cs | 10 +- server/AyaNova/biz/ReportBiz.cs | 7 +- server/AyaNova/util/ReportProcessManager.cs | 178 +++++++++++++----- server/AyaNova/util/ServerBootConfig.cs | 14 +- 7 files changed, 159 insertions(+), 57 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 47589491..d762b8d3 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -51,6 +51,7 @@ //"AYANOVA_DB_CONNECTION": "Server=localhost;Username=postgres;Password=abraxis;Database=AyaNova;CommandTimeout=120;", "AYANOVA_USE_URLS": "http://*:7575;", "AYANOVA_REPORT_RENDERING_TIMEOUT": "5000", + "AYANOVA_REPORT_RENDERING_MAX_INSTANCES": "2", "AYANOVA_FOLDER_USER_FILES": "c:\\temp\\RavenTestData\\userfiles", "AYANOVA_FOLDER_BACKUP_FILES": "c:\\temp\\RavenTestData\\backupfiles", "AYANOVA_FOLDER_TEMPORARY_SERVER_FILES": "c:\\temp\\RavenTestData\\tempfiles", diff --git a/docs/8.0/ayanova/docs/ops-config-environment-variables.md b/docs/8.0/ayanova/docs/ops-config-environment-variables.md index a74adb4c..0d8355bd 100644 --- a/docs/8.0/ayanova/docs/ops-config-environment-variables.md +++ b/docs/8.0/ayanova/docs/ops-config-environment-variables.md @@ -36,6 +36,7 @@ These values can all be specified as an environment variable or as a command lin ## REPORTING - [AYANOVA_REPORT_RENDERING_TIMEOUT](ops-config-report-rendering-timeout.md) +- [AYANOVA_REPORT_RENDERING_MAX_INSTANCES](ops-config-report-rendering-max-instances.md) ## SECURITY diff --git a/docs/8.0/ayanova/docs/ops-config-report-rendering-timeout.md b/docs/8.0/ayanova/docs/ops-config-report-rendering-timeout.md index ec8c383c..c3417d78 100644 --- a/docs/8.0/ayanova/docs/ops-config-report-rendering-timeout.md +++ b/docs/8.0/ayanova/docs/ops-config-report-rendering-timeout.md @@ -21,9 +21,10 @@ If no override is specified AyaNova will use the following default value: This means AyaNova will wait for a prior report to complete for no longer than 30,000 milliseconds or 30 seconds before it will stop that prior report render and start the new one. -## Hard cap +## MINIMUM / MAXIMUM -There is a hard cap of 3 minutes or 180000 milliseconds built into AyaNova so specifying a value greater than 180000 milliseconds will be ignored. +There is a hard cap of 3 minutes or 180000 milliseconds built into AyaNova so specifying a value greater than 180000 milliseconds will be ignored and 3 minutes used instead. +Specifying a value less than 1 second (1000) will be automatically changed to 1000 as the minimum allowed value. ## Overriding diff --git a/server/AyaNova/Controllers/ReportController.cs b/server/AyaNova/Controllers/ReportController.cs index 822ec3fe..9b48231c 100644 --- a/server/AyaNova/Controllers/ReportController.cs +++ b/server/AyaNova/Controllers/ReportController.cs @@ -101,7 +101,7 @@ namespace AyaNova.Api.Controllers ReportBiz biz = ReportBiz.GetBiz(ct, HttpContext); if (!Authorized.HasModifyRole(HttpContext.Items, biz.BizType)) return StatusCode(403, new ApiNotAuthorizedResponse()); - var o = await biz.PutAsync(updatedObject); + var o = await biz.PutAsync(updatedObject); if (o == null) { if (biz.Errors.Exists(z => z.Code == ApiErrorCode.CONCURRENCY_CONFLICT)) @@ -195,6 +195,14 @@ namespace AyaNova.Api.Controllers { if (!serverState.IsOpen) return StatusCode(503, new ApiErrorResponse(serverState.ApiErrorCode, null, serverState.Reason)); + + //return if no free slot + //Note that this *should* normally return a 503 however we're pretty tightly wired into that meaning the server is closed at the client end which + //handles it at a lower level + //returning an OK method here allows the client to handle it at the level of the report dialog rather than the api handler which will short circuit if it was a 503 + if (!Util.ReportRenderManager.RenderSlotAvailable()) + return Ok(ApiOkResponse.Response(new { busy = true, retryafter = DateTime.UtcNow.AddMilliseconds(ServerBootConfig.AYANOVA_REPORT_RENDERING_TIMEOUT) })); + ReportBiz biz = ReportBiz.GetBiz(ct, HttpContext); if (!Authorized.HasReadFullRole(HttpContext.Items, biz.BizType)) return StatusCode(403, new ApiNotAuthorizedResponse()); diff --git a/server/AyaNova/biz/ReportBiz.cs b/server/AyaNova/biz/ReportBiz.cs index c0f40646..1ebb803d 100644 --- a/server/AyaNova/biz/ReportBiz.cs +++ b/server/AyaNova/biz/ReportBiz.cs @@ -434,10 +434,7 @@ namespace AyaNova.Biz log.LogDebug("Initializing report system"); var ReportJSFolderPath = Path.Combine(ServerBootConfig.AYANOVA_CONTENT_ROOT_PATH, "resource", "rpt"); - //Ensure last reporting op has completed - await ReportProcessorManager.EnsureReporterAvailableAsync(log); - - var lo = new LaunchOptions { Headless = true }; + var lo = new LaunchOptions { Headless = true }; bool isWindows = System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Windows); if (!isWindows) { @@ -493,7 +490,7 @@ namespace AyaNova.Biz using (var page = await browser.NewPageAsync()) { //mark this process so it can be cancelled if it times out - ReportProcessorManager.RecordNewReportGeneratorProcess(browser.Process.Id); + ReportRenderManager.AddProcess(browser.Process.Id); try { diff --git a/server/AyaNova/util/ReportProcessManager.cs b/server/AyaNova/util/ReportProcessManager.cs index 3a4fd6af..ac5a113e 100644 --- a/server/AyaNova/util/ReportProcessManager.cs +++ b/server/AyaNova/util/ReportProcessManager.cs @@ -1,5 +1,8 @@ using System; +using System.Collections.Concurrent; +using System.Collections.Generic; using System.Diagnostics; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -12,67 +15,150 @@ namespace AyaNova.Util /// also zombie process issues in linux etc, this just ensures it's safe /// This is triggered when a report is rendered /// - internal static class ReportProcessorManager + internal static class ReportRenderManager { - internal static int ReporterProcessId { get; set; } = -1; - internal static DateTime Started { get; set; } - internal async static Task EnsureReporterAvailableAsync(ILogger log) + /* + Use thread safe concurrent dictionary collection to manage up to AYANOVA_REPORT_RENDERING_MAX_INSTANCES + (it's allowed to go slightly over, this is not ever going to be exactly right under heavy load, but it should always kill the old processes no matter what) + + Render route controller checks for an available slot with the reportprocessormanager first "ProcessSlotAvailable" + if there is no free slot it immediately callse Cleanup which looks for the oldest slot that is over the limit and attempts to shut it down, once shut down it returns true + if there is no free slot and none are over the limit it returns false signifying try again + + + If there *is* a free slot then it passes off to reportbiz as usual + Report biz reserves a slot when launches the browser process with here by adding it to the dictionary "AddProcess(processid)" + When the report is generated it will remove from the slot by calling into here "RemoveProcess(processId)" + Remove from slot here will confirm the process is no longer running and if it is kill it or if it's not remove it from the collection + + expired processes are removed by the act of tryign to get a new slot so in this way it still supports running super long reports overnight for example as long as there is no contention + The other way was by a job that looks for expired processes but that would mean all old jobs would expire all the time so there would be an issue with huge reports never working + + + */ + + //thread safe collection for unordered items, optimized for single thread produce/consume (which is the norm here) but supports multithread produce / consume (which is needed for separate cleanup job) + private static ConcurrentBag _baginstances = new ConcurrentBag(); + + public class ReportRenderInstanceInfo { - Process reportProcess = ReporterProcess(); - if (reportProcess == null) + public int ReporterProcessId { get; set; } + public DateTime Started { get; set; } + + public ReportRenderInstanceInfo(int processId) { - return; + ReporterProcessId = processId; + Started = DateTime.UtcNow; } - //await it's completion in the specified timeout - int HardTimeout = ServerBootConfig.AYANOVA_REPORT_RENDERING_TIMEOUT; - //don't wait forever, hard cap of 3 minutes regardless of setting - if (HardTimeout > 180000) HardTimeout = 180000; - bool keepOnWaiting = true; - while (keepOnWaiting) + } + + + public static bool RenderSlotAvailable() + { + if (_baginstances.Count >= ServerBootConfig.AYANOVA_REPORT_RENDERING_MAX_INSTANCES) { - //don't check continually - await Task.Delay(500); - //check process is still running - if (reportProcess?.HasExited == false) + //check for expired and remove + var Instances = _baginstances.ToArray(); + ReportRenderInstanceInfo oldest = null; + foreach (ReportRenderInstanceInfo i in Instances) { - //time to kill it? - if ((DateTime.UtcNow - Started).TotalMilliseconds > HardTimeout) + if (oldest == null) { - log.LogInformation($"Report processor did not complete in {HardTimeout}ms and will be force stopped"); - reportProcess.Kill(); - keepOnWaiting = false; + oldest = i; + continue; + } + if (i.Started < oldest.Started) oldest = i; + } + if (oldest != null) + { + try + { + var p = Process.GetProcessById(oldest.ReporterProcessId); + if (p != null) + { + //try to kill it + p.Kill(); + if (p?.HasExited == false) return false;//can't kill it so can't free up a slot + } + _baginstances.TryTake(out oldest); + return true;//process that was there is now not there so while not perfect system we will consider it free + + } + catch (ArgumentException) + { + return true;//no process available / not running } } - else - { - log.LogDebug($"EnsureReporterAvailableAsync Reporter processor completed normally"); - keepOnWaiting = false; - } - }; - ReporterProcessId = -1; - Started = DateTime.MinValue; - return; + + } + return true; } - internal static void RecordNewReportGeneratorProcess(int processId) + + internal static void AddProcess(int processId) { - ReporterProcessId = processId; - Started = DateTime.UtcNow; + _baginstances.Add(new ReportRenderInstanceInfo(processId)); } - private static Process ReporterProcess() - { - if (ReporterProcessId == -1) return null; - try - { - return Process.GetProcessById(ReporterProcessId); - } - catch (ArgumentException) - { - return null;//no process available / not running - } - } + + + // internal async static Task EnsureReporterAvailableAsync(ILogger log) + // { + // Process reportProcess = ReporterProcess(); + // if (reportProcess == null) + // { + // return; + // } + // //await it's completion in the specified timeout + // int HardTimeout = ServerBootConfig.AYANOVA_REPORT_RENDERING_TIMEOUT; + // //don't wait forever, hard cap of 3 minutes regardless of setting + // if (HardTimeout > 180000) HardTimeout = 180000; + // bool keepOnWaiting = true; + // while (keepOnWaiting) + // { + // //don't check continually + // await Task.Delay(500); + // //check process is still running + // if (reportProcess?.HasExited == false) + // { + // //time to kill it? + // if ((DateTime.UtcNow - Started).TotalMilliseconds > HardTimeout) + // { + // log.LogInformation($"Report processor did not complete in {HardTimeout}ms and will be force stopped"); + // reportProcess.Kill(); + // keepOnWaiting = false; + // } + // } + // else + // { + // log.LogDebug($"EnsureReporterAvailableAsync Reporter processor completed normally"); + // keepOnWaiting = false; + // } + // }; + // ReporterProcessId = -1; + // Started = DateTime.MinValue; + // return; + // } + + // internal static void RecordNewReportGeneratorProcess(int processId) + // { + // ReporterProcessId = processId; + // Started = DateTime.UtcNow; + // } + + // private static Process ReporterProcess() + // { + // if (ReporterProcessId == -1) return null; + // try + // { + // return Process.GetProcessById(ReporterProcessId); + // } + // catch (ArgumentException) + // { + // return null;//no process available / not running + // } + // } /* diff --git a/server/AyaNova/util/ServerBootConfig.cs b/server/AyaNova/util/ServerBootConfig.cs index 6e70a0e6..26e81eed 100644 --- a/server/AyaNova/util/ServerBootConfig.cs +++ b/server/AyaNova/util/ServerBootConfig.cs @@ -14,7 +14,7 @@ namespace AyaNova.Util //############################################################################################################ //STATIC HARD CODED COMPILE TIME DEFAULTS NOT SET THROUGH CONFIG internal const int FAILED_AUTH_DELAY = 3000;//ms - + //UPLOAD LIMITS 1048576 = 1MiB for testing 10737420000 10737418240 10,737,418,240 internal const long MAX_ATTACHMENT_UPLOAD_BYTES = 10737420000;//slight bit of overage as 10737418241=10GiB internal const long MAX_LOGO_UPLOAD_BYTES = 512000;//500KiB limit @@ -61,6 +61,7 @@ namespace AyaNova.Util internal static string AYANOVA_JWT_SECRET { get; set; } internal static string AYANOVA_USE_URLS { get; set; } internal static int AYANOVA_REPORT_RENDERING_TIMEOUT { get; set; } + internal static int AYANOVA_REPORT_RENDERING_MAX_INSTANCES { get; set; } //DATABASE @@ -161,8 +162,15 @@ namespace AyaNova.Util AYANOVA_JWT_SECRET = config.GetValue("AYANOVA_JWT_SECRET"); - int? nTemp = config.GetValue("AYANOVA_REPORT_RENDERING_TIMEOUT"); - AYANOVA_REPORT_RENDERING_TIMEOUT = (null == nTemp) ? 30000 : (int)nTemp; + //REPORT RENDERING PROCESS CONTROL + int? nTemp = config.GetValue("AYANOVA_REPORT_RENDERING_TIMEOUT"); + AYANOVA_REPORT_RENDERING_TIMEOUT = (null == nTemp) ? 30000 : (int)nTemp;//default is 30 seconds + if (AYANOVA_REPORT_RENDERING_TIMEOUT < 1000) AYANOVA_REPORT_RENDERING_TIMEOUT = 1000; //one second minimum timeout + if (AYANOVA_REPORT_RENDERING_TIMEOUT > 180000) AYANOVA_REPORT_RENDERING_TIMEOUT = 180000; //3 minutes maximum timeout + + nTemp = config.GetValue("AYANOVA_REPORT_RENDERING_MAX_INSTANCES"); + AYANOVA_REPORT_RENDERING_MAX_INSTANCES = (null == nTemp) ? 3 : (int)nTemp; + if (AYANOVA_REPORT_RENDERING_MAX_INSTANCES < 1) AYANOVA_REPORT_RENDERING_MAX_INSTANCES = 1; //minimum instances //DB AYANOVA_DB_CONNECTION = config.GetValue("AYANOVA_DB_CONNECTION");