server/AyaNova/Controllers/ReportController.cs

@@ -196,12 +196,8 @@ namespace AyaNova.Api.Controllers
             if (!serverState.IsOpen)
                 return StatusCode(503, new ApiErrorResponse(serverState.ApiErrorCode, null, serverState.Reason));
 
-            //return if no free slot
-            //Note that this *should* normally return a 503 however this is not an unexpected result and we're pretty tightly wired into that meaning the server is closed at the client end which
-            //handles it at a lower api level
-            //returning an OK method here allows the client to handle it at the level of the report dialog rather than the api handler which will short circuit if it was a 503
-            if (!Util.ReportRenderManager.RenderSlotAvailable(log))
-                return Ok(ApiOkResponse.Response(new { busy = true, retryms = ServerBootConfig.AYANOVA_REPORT_RENDERING_TIMEOUT }));
+            //check for an kill any expired prior renders stuck around
+            Util.ReportRenderManager.KillExpiredRenders(log);
 
             ReportBiz biz = ReportBiz.GetBiz(ct, HttpContext);
             if (!Authorized.HasReadFullRole(HttpContext.Items, biz.BizType))
@@ -213,9 +209,9 @@ namespace AyaNova.Api.Controllers
             var API_URL = $"http://127.0.0.1:{httpConnectionFeature.LocalPort}/api/v8/";
             try
             {
+                var Expires = DateTime.UtcNow.AddMinutes(ServerBootConfig.AYANOVA_REPORT_RENDERING_TIMEOUT);
                 var result = await biz.RenderReport(reportRequest, API_URL);
 
-
                 if (string.IsNullOrWhiteSpace(result))
                     return BadRequest(new ApiErrorResponse(biz.Errors));
                 else

server/AyaNova/biz/ReportBiz.cs

@@ -434,9 +434,7 @@ namespace AyaNova.Biz
             }
 
 
-            //Default timeout for each operation of report generation
-            // var WaitTimeout = new WaitForFunctionOptions() { Timeout = ServerBootConfig.AYANOVA_REPORT_RENDERING_TIMEOUT*10 };
-
+           
             //includeWoItemDescendants?
             reportRequest.IncludeWoItemDescendants = report.IncludeWoItemDescendants;
 

server/AyaNova/util/ReportProcessManager.cs

@@ -6,20 +6,10 @@ using Microsoft.Extensions.Logging;
 namespace AyaNova.Util
 {
 
-    /// <summary>
-    /// Used by reporting system to ensure headless browsers don't hang around in an untimely manner chewing up resources
-    /// needed due to bugs in puppeteersharp where it won't close the browser on timeout properly
-    /// also zombie process issues in linux etc, this just ensures it's safe
-    /// This is triggered when a report is rendered on demand
-    /// in other words demand drives whether it kills long running renders or not
-    /// this is by design to allow a scenario where a super long running report can still be run off hours (for example)
-    /// </summary>
+    //Track processes and kill any that go past their expiry date
     internal static class ReportRenderManager
     {
-        /*        
-            expired processes are removed by the act of trying to get a new slot so in this way it still supports running super long reports overnight for example as long as there is no contention
-            The other way was by a job that looks for expired processes but that would mean all old jobs would expire all the time so there would be an issue with huge reports never working
-        */
+
         //thread safe collection for unordered items, optimized for single thread produce/consume (which is the norm here) but supports multithread produce / consume (which is needed for separate cleanup job)
         internal static ConcurrentBag<ReportRenderInstanceInfo> _baginstances;// = new ConcurrentBag<ReportRenderInstanceInfo>();
 
@@ -34,45 +24,35 @@ namespace AyaNova.Util
             internal int ReporterProcessId { get; set; }
             internal DateTime Expires { get; set; }
 
-            internal ReportRenderInstanceInfo(int processId)
+            internal ReportRenderInstanceInfo(int processId, DateTime expires)
             {
                 ReporterProcessId = processId;
-                Expires = DateTime.UtcNow.AddMilliseconds(ServerBootConfig.AYANOVA_REPORT_RENDERING_TIMEOUT);
+                Expires = expires;
             }
         }
 
 
-        internal static bool RenderSlotAvailable(ILogger log)
+        internal static void KillExpiredRenders(ILogger log)
         {
-            log.LogTrace("RenderSlotAvailable check");
-            //var count = _baginstances.Count;
-            // #if (DEBUG)
-            //             log.LogInformation($"DBG: RenderSlotAvailable check, there are currently {_baginstances.Count} instances in the bag");
-            // #endif
-            if (_baginstances.Count >= ServerBootConfig.AYANOVA_REPORT_RENDERING_MAX_INSTANCES)
+            log.LogTrace("Clear expired render check");
+            //check for expired and remove
+            var Instances = _baginstances.ToArray();
+            var dtNow = DateTime.UtcNow;
+            foreach (ReportRenderInstanceInfo i in Instances)
             {
-                log.LogTrace($"RenderSlotAvailable there are no free report rendering slots available, current count is {_baginstances.Count}, checking for expired slots to force closed");
-                //check for expired and remove
-                var Instances = _baginstances.ToArray();
-                var dtNow = DateTime.UtcNow;
-                foreach (ReportRenderInstanceInfo i in Instances)
+                if (i.Expires < dtNow)
                 {
-                    if (i.Expires < dtNow)
-                    {
-                        // #if (DEBUG)
-                        //                         log.LogInformation($"DBG: RenderSlotAvailable attempting kill of expired process {i.ReporterProcessId}");
-                        // #endif
-                        ForceCloseProcess(i, log);
-                    }
+#if (DEBUG)
+                    log.LogInformation($"DBG: KillExpiredRenders attempting kill of expired process {i.ReporterProcessId}");
+#endif
+                    ForceCloseProcess(i, log);
                 }
             }
-            //allow to continue if there are now fewer than max instances in the bag
-            return _baginstances.Count < ServerBootConfig.AYANOVA_REPORT_RENDERING_MAX_INSTANCES;
         }
 
         internal static bool ForceCloseProcess(ReportRenderInstanceInfo instance, ILogger log)
         {
-            log.LogTrace($"ForceCloseProcess on instance id {instance.ReporterProcessId} started {instance.Expires.ToString()} utc");
+            log.LogTrace($"ForceCloseProcess on report render instance id {instance.ReporterProcessId} exired {instance.Expires.ToString()} utc");
             try
             {
                 var p = Process.GetProcessById(instance.ReporterProcessId);
@@ -83,8 +63,7 @@ namespace AyaNova.Util
                     p.Kill();
                     if (p.HasExited == false)
                     {
-
-                        log.LogDebug($"RenderSlotAvailable oldest slot could not be stopped");
+                        log.LogDebug($"Expired report render instance id {instance.ReporterProcessId} could not be force closed");
                         return false;//can't kill it so can't free up a slot
                     }
                 }
@@ -105,12 +84,12 @@ namespace AyaNova.Util
             }
         }
 
-        internal static void AddProcess(int processId, ILogger log)
+        internal static void AddProcess(int processId, DateTime expires, ILogger log)
         {
             // #if (DEBUG)
             //             log.LogInformation($"DBG: RenderSlotAvailable::AddProcess {processId} in the bag");
             // #endif
-            _baginstances.Add(new ReportRenderInstanceInfo(processId));
+            _baginstances.Add(new ReportRenderInstanceInfo(processId, expires));
 
             // #if (DEBUG)
             //             log.LogInformation($"DBG: RenderSlotAvailable::AddProcess, there are currently {_baginstances.Count} instances in the bag");

server/AyaNova/util/ServerBootConfig.cs

@@ -61,7 +61,7 @@ namespace AyaNova.Util
         internal static string AYANOVA_JWT_SECRET { get; set; }
         internal static string AYANOVA_USE_URLS { get; set; }
         internal static int AYANOVA_REPORT_RENDERING_TIMEOUT { get; set; }
-        internal static int AYANOVA_REPORT_RENDERING_MAX_INSTANCES { get; set; }
+        
 
 
         //DATABASE
@@ -181,15 +181,11 @@ namespace AyaNova.Util
 
             //PROCESS CONTROL
             int? nTemp = config.GetValue<int?>("AYANOVA_REPORT_RENDERING_TIMEOUT");
-            AYANOVA_REPORT_RENDERING_TIMEOUT = (null == nTemp) ? 30000 : (int)nTemp;//default is 30 seconds
-            if (AYANOVA_REPORT_RENDERING_TIMEOUT < 1000) AYANOVA_REPORT_RENDERING_TIMEOUT = 1000; //one second minimum timeout
-            if (AYANOVA_REPORT_RENDERING_TIMEOUT > 180000) AYANOVA_REPORT_RENDERING_TIMEOUT = 180000; //3 minutes maximum timeout
-
-            nTemp = config.GetValue<int?>("AYANOVA_REPORT_RENDERING_MAX_INSTANCES");
-            AYANOVA_REPORT_RENDERING_MAX_INSTANCES = (null == nTemp) ? 3 : (int)nTemp;
-            if (AYANOVA_REPORT_RENDERING_MAX_INSTANCES < 1) AYANOVA_REPORT_RENDERING_MAX_INSTANCES = 1; //minimum instances
-            if (AYANOVA_REPORT_RENDERING_MAX_INSTANCES > 10) AYANOVA_REPORT_RENDERING_MAX_INSTANCES = 10; //Fixed maximum instances
+            AYANOVA_REPORT_RENDERING_TIMEOUT = (null == nTemp) ? 3 : (int)nTemp;//default is 3 minutes
+            if (AYANOVA_REPORT_RENDERING_TIMEOUT < 1) AYANOVA_REPORT_RENDERING_TIMEOUT = 1; //one minute minimum timeout
+            if (AYANOVA_REPORT_RENDERING_TIMEOUT > 15) AYANOVA_REPORT_RENDERING_TIMEOUT = 15; //15 minutes maximum timeout
 
+      
             //DB
             AYANOVA_DB_CONNECTION = config.GetValue<string>("AYANOVA_DB_CONNECTION");
             bTemp = config.GetValue<bool?>("AYANOVA_PERMANENTLY_ERASE_DATABASE");