server/AyaNova/generator/BackgroundService.cs

@@ -0,0 +1,72 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Hosting;
+
+
+
+namespace AyaNova.Generator
+{
+
+
+    //This is a temporary class until .net 2.1 is released
+    //https://docs.microsoft.com/en-us/dotnet/standard/microservices-architecture/multi-container-microservice-net-applications/background-tasks-with-ihostedservice
+
+
+    // Copyright (c) .NET Foundation. Licensed under the Apache License, Version 2.0. 
+    /// <summary>
+    /// Base class for implementing a long running <see cref="IHostedService"/>.
+    /// </summary>
+    public abstract class BackgroundService : IHostedService, IDisposable
+    {
+        private Task _executingTask;
+        private readonly CancellationTokenSource _stoppingCts =
+                                                       new CancellationTokenSource();
+
+        protected abstract Task ExecuteAsync(CancellationToken stoppingToken);
+
+        public virtual Task StartAsync(CancellationToken cancellationToken)
+        {
+            // Store the task we're executing
+            _executingTask = ExecuteAsync(_stoppingCts.Token);
+
+            // If the task is completed then return it, 
+            // this will bubble cancellation and failure to the caller
+            if (_executingTask.IsCompleted)
+            {
+                return _executingTask;
+            }
+
+            // Otherwise it's running
+            return Task.CompletedTask;
+        }
+
+        public virtual async Task StopAsync(CancellationToken cancellationToken)
+        {
+            // Stop called without start
+            if (_executingTask == null)
+            {
+                return;
+            }
+
+            try
+            {
+                // Signal cancellation to the executing method
+                _stoppingCts.Cancel();
+            }
+            finally
+            {
+                // Wait until the task completes or the stop token triggers
+                await Task.WhenAny(_executingTask, Task.Delay(Timeout.Infinite,
+                                                              cancellationToken));
+            }
+
+        }
+
+        public virtual void Dispose()
+        {
+            _stoppingCts.Cancel();
+        }
+    }
+    
+}

server/AyaNova/generator/CoreJobMetricsReport.cs

@@ -0,0 +1,54 @@
+using System;
+using System.Threading.Tasks;
+using App.Metrics;
+using AyaNova.Util;
+using System.Linq;
+
+
+namespace AyaNova.Biz
+{
+
+
+    /// <summary>
+    /// called by Generator to flush metrics to reporter
+    /// 
+    /// </summary>
+    internal static class CoreJobMetricsReport
+    {       
+        private static TimeSpan DO_EVERY_INTERVAL = new TimeSpan(0, 0, 20);//FLUSH EVERY 20 SECONDS
+        private static DateTime lastReportFlushDone = DateTime.MinValue;
+
+        ////////////////////////////////////////////////////////////////////////////////////////////////
+        // DoAsync
+        //
+        public static async Task DoJobAsync()
+        {
+            //https://www.app-metrics.io/
+            IMetrics metrics = (IMetrics)ServiceProviderProvider.Provider.GetService(typeof(IMetrics));
+           
+           //No more quickly than doeveryinterval
+            if (!DateUtil.IsAfterDuration(lastReportFlushDone, DO_EVERY_INTERVAL))
+                return;
+          
+            //RUN ALL REPORTS - FLUSH STATS
+            var mr = (IMetricsRoot)metrics;
+            Task.WaitAll(mr.ReportRunner.RunAllAsync().ToArray());
+
+            lastReportFlushDone = DateTime.UtcNow;
+
+            //just to hide compiler warning for now
+            await Task.CompletedTask;
+
+        }
+
+
+
+
+
+        /////////////////////////////////////////////////////////////////////
+
+    }//eoc
+
+
+}//eons
+

server/AyaNova/generator/CoreJobMetricsSnapshot.cs

@@ -0,0 +1,132 @@
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using System.Diagnostics;
+using Microsoft.Extensions.Logging;
+using App.Metrics;
+using AyaNova.Util;
+using AyaNova.Models;
+
+
+
+namespace AyaNova.Biz
+{
+
+
+    /// <summary>
+    /// called by Generator to gather server metrics and check on things
+    /// See MetricsRegistry for defined metrics 
+    /// 
+    /// </summary>
+    internal static class CoreJobMetricsSnapshot
+    {
+        private static ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("CoreJobMetricsSnapshot");
+
+#if (DEBUG)
+        private static TimeSpan DO_EVERY_INTERVAL = new TimeSpan(0, 1, 0);//DEBUG do a check every 60 seconds
+#else
+        private static TimeSpan DO_EVERY_INTERVAL = new TimeSpan(0, 15, 0);//RELEASE do a check every 15 minutes
+#endif
+
+        private static DateTime lastServerCheckDone = DateTime.MinValue;
+        private static DateTime lastRecordCountCheck = DateTime.MinValue;
+        private static DateTime lastFileCountCheck = DateTime.MinValue;
+
+        ////////////////////////////////////////////////////////////////////////////////////////////////
+        // DoAsync
+        //
+        public static async Task DoJobAsync(AyContext ct)
+        {
+            //https://www.app-metrics.io/
+            IMetrics metrics = (IMetrics)ServiceProviderProvider.Provider.GetService(typeof(IMetrics));
+
+            //This will get triggered roughly every minute (10 seconds in debug), but we don't want to healthcheck that frequently
+            if (!DateUtil.IsAfterDuration(lastServerCheckDone, DO_EVERY_INTERVAL))
+                return;
+
+            log.LogTrace("Starting metrics snapshot");
+
+            //Gather core metrics here
+            var process = Process.GetCurrentProcess();
+
+            //PHYSICAL MEMORY
+            metrics.Measure.Gauge.SetValue(MetricsRegistry.PhysicalMemoryGauge, process.WorkingSet64);
+
+            //PRIVATE BYTES
+            metrics.Measure.Gauge.SetValue(MetricsRegistry.PrivateBytesGauge, process.PrivateMemorySize64);
+
+            //RECORDS IN TABLE
+            //Only do this once per hour
+            if (DateUtil.IsAfterDuration(lastRecordCountCheck, 1))
+            {
+                lastRecordCountCheck = DateTime.UtcNow;
+                log.LogTrace("Counting table records");
+
+
+                //Get a count of important tables in db
+                List<string> allTableNames = DbUtil.GetAllTablenames();
+
+                //Skip some tables as they are internal and / or only ever have one record
+                List<string> skipTableNames = new List<string>();
+                skipTableNames.Add("alicense");
+                skipTableNames.Add("aschemaversion");
+
+                foreach (string table in allTableNames)
+                {
+                    if (!skipTableNames.Contains(table))
+                    {
+                        var tags = new MetricTags("TableTagKey", table);
+                        metrics.Measure.Gauge.SetValue(MetricsRegistry.DBRecordsGauge, tags, DbUtil.CountOfRecords(table));
+                    }
+                }
+            }
+
+            //JOB COUNTS (DEAD, RUNNING, COMPLETED, SLEEPING)
+
+            foreach (JobStatus stat in Enum.GetValues(typeof(JobStatus)))
+            {
+                var jobtag = new MetricTags("JobStatus", stat.ToString());
+                metrics.Measure.Gauge.SetValue(MetricsRegistry.JobsGauge, jobtag, await JobsBiz.GetCountForJobStatusAsync(ct, stat));
+            }
+
+
+
+            //FILES ON DISK
+            //Only do this once per hour
+            if (DateUtil.IsAfterDuration(lastFileCountCheck, 1))
+            {
+                lastFileCountCheck = DateTime.UtcNow;
+                log.LogTrace("Files on disk information");
+                var UtilFilesInfo = FileUtil.GetUtilityFolderSizeInfo();
+                var UserFilesInfo = FileUtil.GetAttachmentFolderSizeInfo();
+
+                var mtag = new MetricTags("File type", "Business object files");
+                metrics.Measure.Gauge.SetValue(MetricsRegistry.FileCountGauge, mtag, UserFilesInfo.FileCountWithChildren);
+                metrics.Measure.Gauge.SetValue(MetricsRegistry.FileSizeGauge, mtag, UserFilesInfo.SizeWithChildren);
+
+                mtag = new MetricTags("File type", "OPS files");
+                metrics.Measure.Gauge.SetValue(MetricsRegistry.FileCountGauge, mtag, UtilFilesInfo.FileCountWithChildren);
+                metrics.Measure.Gauge.SetValue(MetricsRegistry.FileSizeGauge, mtag, UtilFilesInfo.SizeWithChildren);
+
+            }
+
+
+            lastServerCheckDone = DateTime.UtcNow;
+
+
+            //just to hide compiler warning for now
+            await Task.CompletedTask;
+
+        }
+
+
+
+
+
+        /////////////////////////////////////////////////////////////////////
+
+    }//eoc
+
+
+}//eons
+

server/AyaNova/generator/CoreJobSweeper.cs

@@ -0,0 +1,127 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.JsonPatch;
+using Microsoft.Extensions.Logging;
+using EnumsNET;
+using AyaNova.Util;
+using AyaNova.Api.ControllerHelpers;
+using AyaNova.Biz;
+using AyaNova.Models;
+
+
+namespace AyaNova.Biz
+{
+
+
+    /// <summary>
+    /// JobSweeper - called by Generator to clean out old jobs that are completed and their logs
+    /// 
+    /// </summary>
+    internal static class CoreJobSweeper
+    {
+        private static ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("CoreJobSweeper");
+        private static DateTime lastSweep = DateTime.MinValue;
+        private static TimeSpan SWEEP_EVERY_INTERVAL = new TimeSpan(0, 30, 0);
+        private static TimeSpan SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN = new TimeSpan(24, 0, 0);//24 hours
+        private static TimeSpan FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN = new TimeSpan(14, 0, 0, 0);//14 days (gives people time to notice and look into it)
+        private static TimeSpan RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN = new TimeSpan(24, 0, 0);//24 hours (time running jobs are allowed to sit in "running" state before considered failed)
+
+        ////////////////////////////////////////////////////////////////////////////////////////////////
+        // DoSweep
+        //
+        public static async Task DoSweepAsync(AyContext ct)
+        {
+
+
+            //This will get triggered roughly every minute, but we don't want to sweep that frequently
+            if (DateTime.UtcNow - lastSweep < SWEEP_EVERY_INTERVAL)
+                return;
+
+            log.LogTrace("Sweep starting");
+
+            //SWEEP SUCCESSFUL JOBS
+            //calculate cutoff to delete 
+            DateTime dtDeleteCutoff = DateTime.UtcNow - SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
+            await sweepAsync(ct, dtDeleteCutoff, JobStatus.Completed);
+
+            //SWEEP FAILED JOBS
+            //calculate cutoff to delete 
+            dtDeleteCutoff = DateTime.UtcNow - FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
+            await sweepAsync(ct, dtDeleteCutoff, JobStatus.Failed);
+
+
+            //KILL STUCK JOBS
+            //calculate cutoff to delete 
+            DateTime dtRunningDeadline = DateTime.UtcNow - RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN;
+            await killStuckJobsAsync(ct, dtRunningDeadline);
+
+            lastSweep = DateTime.UtcNow;
+        }
+
+
+        private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus)
+        {
+
+            //Get the deleteable succeeded jobs list
+            var jobs = await ct.OpsJob
+                   .AsNoTracking()
+                   .Where(c => c.Created < dtDeleteCutoff && c.JobStatus == jobStatus)
+                   .OrderBy(m => m.Created)
+                   .ToListAsync();
+
+            log.LogTrace($"SweepAsync processing: cutoff={dtDeleteCutoff.ToString()}, for {jobs.Count.ToString()} jobs of status {jobStatus.ToString()}");
+
+            foreach (OpsJob j in jobs)
+            {
+                try
+                {
+                    await JobsBiz.DeleteJobAndLogAsync(j.GId, ct);
+                }
+                catch (Exception ex)
+                {
+                    log.LogError(ex, "sweepAsync exception calling JobsBiz.DeleteJobAndLogAsync");
+                    //for now just throw it but this needs to be removed when logging added and better handling 
+                    throw (ex);
+                }
+            }
+        }
+
+
+        /// <summary>
+        /// Kill jobs that have been stuck in "running" state for too long
+        /// </summary>
+        /// <param name="ct"></param>
+        /// <param name="dtRunningDeadline"></param>
+        /// <returns></returns>
+        private static async Task killStuckJobsAsync(AyContext ct, DateTime dtRunningDeadline)
+        {
+            //Get the deleteable succeeded jobs list
+            var jobs = await ct.OpsJob
+                   .AsNoTracking()
+                   .Where(c => c.Created < dtRunningDeadline && c.JobStatus == JobStatus.Running)
+                   .OrderBy(m => m.Created)
+                   .ToListAsync();
+
+            log.LogTrace($"killStuckJobsAsync processing: cutoff={dtRunningDeadline.ToString()}, for {jobs.Count.ToString()} jobs of status {JobStatus.Running.ToString()}");
+
+            foreach (OpsJob j in jobs)
+            {
+                //OPSMETRIC
+                JobsBiz.LogJob(j.GId, "Job took too long to run - setting to failed", ct);
+                log.LogError($"Job found job stuck in running status and set to failed: deadline={dtRunningDeadline.ToString()}, jobId={j.GId.ToString()}, jobname={j.Name}, jobtype={j.JobType.ToString()}, jobObjectType={j.ObjectType.ToString()}, jobObjectId={j.ObjectId.ToString()}");
+                JobsBiz.UpdateJobStatus(j.GId, JobStatus.Failed, ct);
+            }
+        }
+
+
+        /////////////////////////////////////////////////////////////////////
+
+    }//eoc
+
+
+}//eons
+

server/AyaNova/generator/Generate.cs

@@ -0,0 +1,124 @@
+using System.Threading;
+using System;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.DependencyInjection;
+using AyaNova.Models;
+using AyaNova.Api.ControllerHelpers;
+using AyaNova.Biz;
+
+namespace AyaNova.Generator
+{
+
+    //Implemented from a example here
+    //https://docs.microsoft.com/en-us/dotnet/standard/microservices-architecture/multi-container-microservice-net-applications/background-tasks-with-ihostedservice
+
+
+    /*
+    TODO: Generator tasks that should happen:
+    - Periodically erase any temp files written to userfiles root (attachments temp files) that are older than a day
+        - These files should be normally erased within seconds after uploading and processing into their permanent folder but shit will go wrong
+     */
+
+    public class GeneratorService : BackgroundService
+    {
+        private readonly ILogger<GeneratorService> log;
+        // private readonly AyContext ct;
+        // private readonly ApiServerState serverState;
+
+        private readonly IServiceProvider provider;
+
+#if(DEBUG)
+        private const int GENERATE_SECONDS = 10;
+#else
+        private const int GENERATE_SECONDS = 60;
+#endif
+
+        // public GeneratorService(ILogger<GeneratorService> logger, AyContext dbcontext, ApiServerState apiServerState)
+        // {
+        //     ct = dbcontext;
+        //     log = logger;
+        //     serverState = apiServerState;
+        // }
+
+
+        public GeneratorService(ILogger<GeneratorService> logger, IServiceProvider serviceProvider)
+        {
+            // ct = dbcontext;
+            provider = serviceProvider;
+
+            log = logger;
+            //  serverState = apiServerState;
+        }
+
+
+
+
+
+
+        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+        {
+            //don't immediately run the generator stuff on boot
+            bool justStarted = false;
+
+            log.LogDebug($"GeneratorService is starting.");
+
+            stoppingToken.Register(() =>
+                    log.LogDebug($" GeneratorService background task is stopping."));
+
+            while (!stoppingToken.IsCancellationRequested)
+            {
+               
+                if (!justStarted)
+                {
+                    log.LogDebug($"GeneratorService task doing background work.");
+
+                    using (IServiceScope scope = provider.CreateScope())
+                    {
+                        AyContext ct = scope.ServiceProvider.GetRequiredService<AyContext>();
+                        ApiServerState serverState = scope.ServiceProvider.GetRequiredService<ApiServerState>();
+
+                        if (!serverState.IsOpen)
+                        {
+                            log.LogDebug($"GeneratorService: ServerState is closed returning without processing jobs, will try again next iteration");
+                        }
+
+                        //=================================================================
+                        try
+                        {
+                            await JobsBiz.ProcessJobsAsync(ct);                           
+                        }
+                        catch (Exception ex)
+                        {
+                            log.LogError("Generate::ProcessJobs resulted in exception error ", ex);
+                        
+                        }
+                        //=================================================================
+                    }
+                }
+                await Task.Delay((GENERATE_SECONDS * 1000), stoppingToken);
+                justStarted = false;
+            }
+
+            log.LogDebug($"GeneratorService background task is stopping.");
+
+        }
+
+        //originally but kept getting compiler error
+        //  public override async Task StopAsync(CancellationToken stoppingToken)
+        //         {
+        //             log.LogDebug($"GeneratorService StopAsync triggered.");
+
+        //             // Run your graceful clean-up actions
+        //         }
+
+
+        public override Task StopAsync(CancellationToken stoppingToken)
+        {
+            log.LogDebug($"GeneratorService StopAsync triggered.");
+            return Task.FromResult(0);
+            // Run your graceful clean-up actions
+        }
+    }
+
+}