admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Report rendering handler to prevent linux from crashing on too much reporting at once

Browse Source
This commit is contained in:
John Cardinal
2021-10-27 00:11:57 +00:00
parent 35c39c30a7
commit ae2b42065a
7 changed files with 103 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
server/AyaNova/Controllers/ServerStateController.cs
View File

@@ -183,6 +183,7 @@ namespace AyaNova.Api.Controllers
AYANOVA_DEFAULT_TRANSLATION = ServerBootConfig.AYANOVA_DEFAULT_TRANSLATION,
AYANOVA_USE_URLS = ServerBootConfig.AYANOVA_USE_URLS,
AYANOVA_DB_CONNECTION = DbUtil.PasswordRedactedConnectionString(ServerBootConfig.AYANOVA_DB_CONNECTION),
AYANOVA_REPORT_RENDERING_TIMEOUT = ServerBootConfig.AYANOVA_REPORT_RENDERING_TIMEOUT,
AYANOVA_FOLDER_USER_FILES = ServerBootConfig.AYANOVA_FOLDER_USER_FILES,
AYANOVA_FOLDER_BACKUP_FILES = ServerBootConfig.AYANOVA_FOLDER_BACKUP_FILES,
AYANOVA_FOLDER_TEMPORARY_SERVER_FILES = ServerBootConfig.AYANOVA_FOLDER_TEMPORARY_SERVER_FILES,

21
server/AyaNova/biz/ReportBiz.cs
View File

@@ -417,7 +417,7 @@ namespace AyaNova.Biz
//Default timeout for each operation of report generation
var WaitTimeout = new WaitForFunctionOptions() { Timeout = ServerBootConfig.REPORT_RENDERING_OPERATION_TIMEOUT };
var WaitTimeout = new WaitForFunctionOptions() { Timeout = ServerBootConfig.AYANOVA_REPORT_RENDERING_TIMEOUT };
//includeWoItemDescendants?
reportRequest.IncludeWoItemDescendants = report.IncludeWoItemDescendants;
@@ -435,7 +435,7 @@ namespace AyaNova.Biz
var ReportJSFolderPath = Path.Combine(ServerBootConfig.AYANOVA_CONTENT_ROOT_PATH, "resource", "rpt");
//Ensure last reporting op has completed
ReportingProcessCache.EnsureReporterAvailable(ServerBootConfig.REPORT_RENDERING_OPERATION_TIMEOUT);
await ReportProcessorManager.EnsureReporterAvailableAsync(log);
var lo = new LaunchOptions { Headless = true };
bool isWindows = System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Windows);
@@ -493,7 +493,7 @@ namespace AyaNova.Biz
using (var page = await browser.NewPageAsync())
{
//mark this process so it can be cancelled if it times out
ReportingProcessCache.FlagNewProcess(browser.Process.Id);
ReportProcessorManager.RecordNewReportGeneratorProcess(browser.Process.Id);
try
{
@@ -616,7 +616,6 @@ namespace AyaNova.Biz
await page.WaitForExpressionAsync($"ayPreRender({ReportDataObject})", WaitTimeout);
//compile the template
//NOTE: TIMEOUT?
log.LogDebug($"Calling Handlebars.compile...");
var compileScript = $"Handlebars.compile(`{report.Template}`)(PreParedReportDataObject);";
var compiledHTML = await page.EvaluateExpressionAsync<string>(compileScript);
@@ -720,11 +719,6 @@ namespace AyaNova.Biz
}
catch
{
//NOTE: in future may need to kill the chromium process if it's found to be hanging around
//my preliminary thinking is to keep track of the browser.process.processid above and check if still present and zap if so
//maybe *from* the controller (return pid as a return property from here? Kill it if still hanging around)
//https://stackoverflow.com/questions/1642231/how-to-kill-a-c-sharp-process
//This is the error when a helper is used on the template but doesn't exist:
//Evaluation failed: d
//(it might also mean other things wrong with template)
@@ -741,15 +735,6 @@ namespace AyaNova.Biz
{
log.LogDebug($"Closing browser");
await browser.CloseAsync();
//this probably isn't absolutely necessary but insurance
//bugbug: crashes linux?
// var process = System.Diagnostics.Process.GetProcessById(ChromiumProcessID);
// if (ChromiumProcessID > 0 && process?.HasExited == false)
// {
// log.LogError($"Error during render, Chromium process (pid {ChromiumProcessID}) still active, forcing it to stop now");
// process.Kill();
// }
}
}
}

43
server/AyaNova/util/ReportingProcessCache.cs → server/AyaNova/util/ReportProcessManager.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Diagnostics;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
namespace AyaNova.Util
{
@@ -11,18 +12,50 @@ namespace AyaNova.Util
/// also zombie process issues in linux etc, this just ensures it's safe
/// This is triggered when a report is rendered
/// </summary>
internal static class ReportingProcessCache
internal static class ReportProcessorManager
{
internal static int ReporterProcessId { get; set; } = -1;
internal static DateTime Started { get; set; }
internal static void EnsureReporterAvailable(int timeOutMilliSeconds)
internal async static Task EnsureReporterAvailableAsync(ILogger log)
{
if (ReporterProcess() == null) return;
Process reportProcess = ReporterProcess();
if (reportProcess == null)
{
return;
}
//await it's completion in the specified timeout
int HardTimeout = ServerBootConfig.AYANOVA_REPORT_RENDERING_TIMEOUT;
//don't wait forever, hard cap of 3 minutes regardless of setting
if (HardTimeout > 180000) HardTimeout = 180000;
bool keepOnWaiting = true;
while (keepOnWaiting)
{
//don't check continually
await Task.Delay(500);
//check process is still running
if (reportProcess?.HasExited == false)
{
//time to kill it?
if ((DateTime.UtcNow - Started).TotalMilliseconds > HardTimeout)
{
log.LogInformation($"Report processor did not complete in {HardTimeout}ms and will be force stopped");
reportProcess.Kill();
keepOnWaiting = false;
}
}
else
{
log.LogDebug($"EnsureReporterAvailableAsync Reporter processor completed normally");
keepOnWaiting = false;
}
};
ReporterProcessId = -1;
Started = DateTime.MinValue;
return;
}
internal static void FlagNewProcess(int processId)
internal static void RecordNewReportGeneratorProcess(int processId)
{
ReporterProcessId = processId;
Started = DateTime.UtcNow;

7
server/AyaNova/util/ServerBootConfig.cs
View File

@@ -14,7 +14,7 @@ namespace AyaNova.Util
//############################################################################################################
//STATIC HARD CODED COMPILE TIME DEFAULTS NOT SET THROUGH CONFIG
internal const int FAILED_AUTH_DELAY = 3000;//ms
internal const int REPORT_RENDERING_OPERATION_TIMEOUT = 20000;//ms
//UPLOAD LIMITS 1048576 = 1MiB for testing 10737420000 10737418240 10,737,418,240
internal const long MAX_ATTACHMENT_UPLOAD_BYTES = 10737420000;//slight bit of overage as 10737418241=10GiB
internal const long MAX_LOGO_UPLOAD_BYTES = 512000;//500KiB limit
@@ -60,6 +60,8 @@ namespace AyaNova.Util
//API
internal static string AYANOVA_JWT_SECRET { get; set; }
internal static string AYANOVA_USE_URLS { get; set; }
internal static int AYANOVA_REPORT_RENDERING_TIMEOUT { get; set; }
//DATABASE
internal static string AYANOVA_DB_CONNECTION { get; set; }
@@ -159,6 +161,9 @@ namespace AyaNova.Util
AYANOVA_JWT_SECRET = config.GetValue<string>("AYANOVA_JWT_SECRET");
int? nTemp = config.GetValue<int?>("AYANOVA_REPORT_RENDERING_TIMEOUT");
AYANOVA_REPORT_RENDERING_TIMEOUT = (null == nTemp) ? 30000 : (int)nTemp;
//DB
AYANOVA_DB_CONNECTION = config.GetValue<string>("AYANOVA_DB_CONNECTION");
bTemp = config.GetValue<bool?>("AYANOVA_PERMANENTLY_ERASE_DATABASE");