admin/sockeye
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Browse Source
This commit is contained in:
John Cardinal
2022-12-16 06:01:23 +00:00
parent 26c2ae5cc9
commit effd96143f
310 changed files with 48715 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
server/generator/BackgroundService.cs Normal file
View File

@@ -0,0 +1,69 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
namespace Sockeye.Generator
{
// Copyright (c) .NET Foundation. Licensed under the Apache License, Version 2.0.
/// <summary>
/// Base class for implementing a long running <see cref="IHostedService"/>.
/// </summary>
public abstract class BackgroundService : IHostedService, IDisposable
{
private Task _executingTask;
private readonly CancellationTokenSource _stoppingCts =
new CancellationTokenSource();
protected abstract Task ExecuteAsync(CancellationToken stoppingToken);
public virtual Task StartAsync(CancellationToken cancellationToken)
{
// Store the task we're executing
_executingTask = ExecuteAsync(_stoppingCts.Token);
// If the task is completed then return it,
// this will bubble cancellation and failure to the caller
if (_executingTask.IsCompleted)
{
return _executingTask;
}
// Otherwise it's running
return Task.CompletedTask;
}
public virtual async Task StopAsync(CancellationToken cancellationToken)
{
// Stop called without start
if (_executingTask == null)
{
return;
}
try
{
// Signal cancellation to the executing method
_stoppingCts.Cancel();
}
finally
{
// Wait until the task completes or the stop token triggers
await Task.WhenAny(_executingTask, Task.Delay(Timeout.Infinite,
cancellationToken));
}
}
public virtual void Dispose()
{
_stoppingCts.Cancel();
}
}
}

48
server/generator/CoreIntegrationLogSweeper.cs Normal file
View File

@@ -0,0 +1,48 @@
using System;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Sockeye.Models;
namespace Sockeye.Biz
{
/// <summary>
/// Clear out old integration log data
/// </summary>
internal static class CoreIntegrationLogSweeper
{
private static ILogger log = Sockeye.Util.ApplicationLogging.CreateLogger("CoreIntegrationLogSweeper");
private static DateTime lastSweep = DateTime.MinValue;
private static TimeSpan DELETE_AFTER_AGE = new TimeSpan(90, 0, 0, 0);//The same typical 90 days as everything uses
private static TimeSpan SWEEP_EVERY_INTERVAL = new TimeSpan(8, 0, 0);//once every 8 hours, three times a day
////////////////////////////////////////////////////////////////////////////////////////////////
// DoSweep
//
public static async Task DoWorkAsync()
{
//This will get triggered roughly every minute, but we don't want to sweep that frequently
if (DateTime.UtcNow - lastSweep < SWEEP_EVERY_INTERVAL)
return;
DateTime dtDeleteCutoff = DateTime.UtcNow - DELETE_AFTER_AGE;
DateTime dtPastEventCutoff = DateTime.UtcNow - SWEEP_EVERY_INTERVAL;
log.LogDebug("Sweep starting");
using (AyContext ct = Sockeye.Util.ServiceProviderProvider.DBContext)
{
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aintegrationlog where created < {dtDeleteCutoff}");
}
lastSweep = DateTime.UtcNow;
}
/////////////////////////////////////////////////////////////////////
}//eoc
}//eons

156
server/generator/CoreJobBackup.cs Normal file
View File

@@ -0,0 +1,156 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Sockeye.Models;
using Sockeye.Util;
using System.IO;
using System.Runtime.InteropServices;
namespace Sockeye.Biz
{
/// <summary>
/// Backup
///
/// </summary>
internal static class CoreJobBackup
{
private static ILogger log = Sockeye.Util.ApplicationLogging.CreateLogger("CoreJobBackup");
private static bool BackupIsRunning = false;
private const int MAXIMUM_MS_ALLOWED_FOR_PROCESSING = 10 * 60 * 1000;//wild assed guess 10 minutes maximum to run backup command, ever
////////////////////////////////////////////////////////////////////////////////////////////////
// BACK-THE-FUCK-UP
//
public static async Task DoWorkAsync(bool OnDemand = false)
{
if (BackupIsRunning) return;
if (!OnDemand)
{
log.LogTrace("Checking if backup should run");
if (!ServerGlobalOpsSettingsCache.Backup.Active)
{
log.LogDebug("Automatic backup is set to INACTIVE - not backing up");
return;
}
if (DateTime.UtcNow < ServerGlobalOpsSettingsCache.NextBackup)
{
log.LogTrace("Not past backup time yet");
return;
}
}
Sockeye.Api.ControllerHelpers.ApiServerState apiServerState = null;
try
{
BackupIsRunning = true;
//LOCK DOWN SERVER
apiServerState = (Sockeye.Api.ControllerHelpers.ApiServerState)ServiceProviderProvider.Provider.GetService(typeof(Sockeye.Api.ControllerHelpers.ApiServerState));
apiServerState.SetClosed("BACKUP RUNNING");
var jobstartmessage = $"LT:Backup LT:StartJob {(OnDemand ? "manual / on demand" : "scheduled")} ";
await JobsBiz.LogJobAsync(Guid.Empty, jobstartmessage);
DateTime dtStartBackup = DateTime.Now;
log.LogDebug("Backup starting");
var DemandFileNamePrepend = OnDemand ? "manual-" : string.Empty;
//*************
//DO DATA BACKUP
//build command
//this is valid on windows
//C:\data\code\PostgreSQLPortable_12.0\App\PgSQL\bin\pg_dump --dbname=postgresql://postgres:raven@127.0.0.1:5432/Sockeye -Fc > huge_new.backup
// await JobsBiz.LogJobAsync(Guid.Empty, $"Data backup starting");
Npgsql.NpgsqlConnectionStringBuilder PostgresConnectionString = new Npgsql.NpgsqlConnectionStringBuilder(ServerBootConfig.SOCKEYE_DB_CONNECTION);
var DBNameParameter = $"--dbname=postgresql://{PostgresConnectionString.Username}:{PostgresConnectionString.Password}@{PostgresConnectionString.Host}:{PostgresConnectionString.Port}/{PostgresConnectionString.Database}";
var DataBackupFile = $"{DemandFileNamePrepend}db-{FileUtil.GetSafeDateFileName()}.backup";//presentation issue so don't use UTC for this one
DataBackupFile = FileUtil.GetFullPathForBackupFile(DataBackupFile);
var BackupUtilityCommand = "pg_dump";
if (!string.IsNullOrWhiteSpace(ServerBootConfig.SOCKEYE_BACKUP_PG_DUMP_PATH))
BackupUtilityCommand = Path.Combine(ServerBootConfig.SOCKEYE_BACKUP_PG_DUMP_PATH, BackupUtilityCommand);
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
//put quotes around paths if spaces in them
if (BackupUtilityCommand.Contains(' '))
{
BackupUtilityCommand = $"\"{BackupUtilityCommand}\"";
}
if (DataBackupFile.Contains(' '))
{
DataBackupFile = $"\"{DataBackupFile}\"";
}
}
var Arguments = $"{DBNameParameter} -Fc > {DataBackupFile}";
var Result = RunProgram.Run(BackupUtilityCommand, Arguments, log, MAXIMUM_MS_ALLOWED_FOR_PROCESSING);
if (!string.IsNullOrWhiteSpace(Result))
{
var msg = $"Error during data backup \"{Result}\"";
await JobsBiz.LogJobAsync(Guid.Empty, msg);
log.LogError($"BACKUP ERROR: {Result}");
await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.BackupStatus, msg, "Backup");
}
else
{
log.LogDebug("Backup of database completed OK");
//DO FILE BACKUP IF ATTACHMENTS BACKED UP
if (ServerGlobalOpsSettingsCache.Backup.BackupAttachments)
{
await JobsBiz.LogJobAsync(Guid.Empty, $"LT:Backup LT:Attachments");
FileUtil.BackupAttachments(DemandFileNamePrepend);
log.LogDebug("Backup of file attachments completed OK");
}
//PRUNE DATA BACKUP SETS NOT KEPT
await JobsBiz.LogJobAsync(Guid.Empty, $"LT:BackupDeleteOld");
FileUtil.DatabaseBackupCleanUp(ServerGlobalOpsSettingsCache.Backup.BackupSetsToKeep);
//v.next - COPY TO ONLINE STORAGE
//***************
log.LogDebug("Backup completed");
var duration = DateTime.Now - dtStartBackup;
await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.BackupStatus, $"Backup ({(OnDemand ? "manual / on demand" : "scheduled")}) completed successfully, duration (h:m:s.ms): {duration.ToString()}, server re-opened", "Backup");
}
}
catch (Exception ex)
{
await JobsBiz.LogJobAsync(Guid.Empty, "LT:JobFailed");
await JobsBiz.LogJobAsync(Guid.Empty, ExceptionUtil.ExtractAllExceptionMessages(ex));
log.LogError(ex, "Backup failed");
await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.BackupStatus, "Backup failed", "Backup", ex);
throw;
}
finally
{
//bump the backup date if automatic backup
if (!OnDemand)
ServerGlobalOpsSettingsCache.SetNextBackup();
apiServerState.ResumePriorState();
BackupIsRunning = false;
await JobsBiz.LogJobAsync(Guid.Empty, "LT:JobCompleted");
}
}
/////////////////////////////////////////////////////////////////////
}//eoc
}//eons

285
server/generator/CoreJobCustomerNotify.cs Normal file
View File

@@ -0,0 +1,285 @@
using System;
using System.Linq;
using System.Collections.Generic;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Sockeye.Models;
using Sockeye.Util;
using Newtonsoft.Json.Linq;
using System.Text.RegularExpressions;
namespace Sockeye.Biz
{
/// <summary>
/// Notification processor
/// turn notifyEvent records into inappnotification records for in app viewing and / or deliver smtp notifications seperately
///
/// </summary>
internal static class CoreJobCustomerNotify
{
private static bool NotifyIsRunning = false;
private static ILogger log = Sockeye.Util.ApplicationLogging.CreateLogger("CoreJobCustomerNotify");
private static DateTime lastRun = DateTime.MinValue;
#if (DEBUG)
private static TimeSpan RUN_EVERY_INTERVAL = new TimeSpan(0, 0, 21);//no more frequently than once every 20 seconds
#else
private static TimeSpan RUN_EVERY_INTERVAL = new TimeSpan(0, 1, 1);//no more frequently than once every 1 minute
#endif
////////////////////////////////////////////////////////////////////////////////////////////////
// DoSweep
//
public static async Task DoWorkAsync()
{
log.LogTrace("Checking if CustomerNotify should run");
if (NotifyIsRunning)
{
log.LogTrace("CustomerNotify is running already exiting this cycle");
return;
}
//This will get triggered roughly every minute, but we don't want to deliver that frequently
if (DateTime.UtcNow - lastRun < RUN_EVERY_INTERVAL)
{
log.LogTrace($"CustomerNotify ran less than {RUN_EVERY_INTERVAL} ago, exiting this cycle");
return;
}
try
{
NotifyIsRunning = true;
log.LogDebug("CustomerNotify set to RUNNING state and starting now");
using (AyContext ct = Sockeye.Util.ServiceProviderProvider.DBContext)
{
var customerevents = await ct.CustomerNotifyEvent.AsNoTracking().ToListAsync();
log.LogDebug($"Found {customerevents.Count} CustomerNotifyEvents to examine for potential delivery");
//iterate and deliver
foreach (var customernotifyevent in customerevents)
{
//no notifications for inactive users, just delete it as if it was delivered
var CustInfo = await ct.Customer.AsNoTracking().Where(x => x.Id == customernotifyevent.CustomerId).Select(x => new { x.Name, x.Active, x.Tags, x.EmailAddress }).FirstOrDefaultAsync();
if (!CustInfo.Active)
{
log.LogDebug($"Inactive Customer {CustInfo.Name}, removing notify rather than delivering it: {customernotifyevent}");
ct.CustomerNotifyEvent.Remove(customernotifyevent);
await ct.SaveChangesAsync();
continue;
}
if (string.IsNullOrWhiteSpace(CustInfo.EmailAddress))
{
log.LogDebug($"Customer {CustInfo.Name} has no email address, removing notify rather than delivering it: {customernotifyevent}");
ct.CustomerNotifyEvent.Remove(customernotifyevent);
await ct.SaveChangesAsync();
continue;
}
//Get subscription for delivery
var Subscription = await ct.CustomerNotifySubscription.AsNoTracking().FirstOrDefaultAsync(x => x.Id == customernotifyevent.CustomerNotifySubscriptionId);
//NOTE: There is no need to separate out future delivery and immediate delivery because
// All events have an event date, it's either immediate upon creation or it's future
// but not all events have an age value including ones with future event dates,
// and the default agevalue and advancenotice are both zero regardless so the block below works for either future or immediate deliveries
var deliverAfter = customernotifyevent.EventDate + Subscription.AgeValue - Subscription.AdvanceNotice;
if (deliverAfter < DateTime.UtcNow)
{
//Do the delivery, it's kosher
await DeliverCustomerNotificationSMTP(customernotifyevent, Subscription, CustInfo.EmailAddress, ct);
}
}
}
}
catch (Exception ex)
{
log.LogError(ex, $"Error processing customer notification event");
DbUtil.HandleIfDatabaseUnavailableTypeException(ex);
}
finally
{
log.LogDebug("CustomerNotify is done setting to not running state and tagging lastRun timestamp");
lastRun = DateTime.UtcNow;
NotifyIsRunning = false;
}
}
//===
private static async Task DeliverCustomerNotificationSMTP(CustomerNotifyEvent ne, CustomerNotifySubscription subscription, string deliveryAddress, AyContext ct)
{
var DeliveryLogItem = new CustomerNotifyDeliveryLog()
{
Processed = DateTime.UtcNow,
ObjectId = ne.ObjectId,
CustomerNotifySubscriptionId = ne.CustomerNotifySubscriptionId,
Fail = false
};
try
{
log.LogDebug($"DeliverCustomerNotificationSMTP delivering notify event: {ne}");
if (string.IsNullOrWhiteSpace(deliveryAddress))
{
DeliveryLogItem.Fail = true;
DeliveryLogItem.Error = $"No email address provided for smtp delivery; event: {ne}";
}
else
{
if (!ServerGlobalOpsSettingsCache.Notify.SmtpDeliveryActive)
{
await NotifyEventHelper.AddOpsProblemEvent($"Email notifications are set to OFF at server, unable to send Customer email notification for this event:{ne}");
log.LogInformation($"** WARNING: SMTP notification is currently set to Active=False; unable to deliver Customer email notification, [CustomerId={ne.CustomerId}, Customer Notify subscription={ne.CustomerNotifySubscriptionId}]. Change this setting or remove all Customer notifications if this is permanent **");
DeliveryLogItem.Fail = true;
DeliveryLogItem.Error = $"Email notifications are set to OFF at server, unable to send Customer email notification for this event: {ne}";
}
else
{
//BUILD SUBJECT AND BODY FROM TOKENS IF REQUIRED
var Subject = subscription.Subject;
var Body = subscription.Template;
// if (Subject.Contains("{{") || Body.Contains("{{"))
// {
// //fetch the object with viz fields for easy templatization
// switch (ne.SockType)
// {
// }
// }
IMailer m = Sockeye.Util.ServiceProviderProvider.Mailer;
//generate report if applicable
bool isReportableEvent = false;
// switch (ne.EventType)
// {
// case NotifyEventType.QuoteStatusChange:
// case NotifyEventType.WorkorderCompleted:
// case NotifyEventType.WorkorderStatusChange:
// isReportableEvent = true;
// break;
// }
if (isReportableEvent && subscription.LinkReportId != null)
{
long subTranslationId = (long)subscription.TranslationId;
ReportBiz biz = new ReportBiz(ct, 1, subTranslationId, AuthorizationRoles.BizAdmin);
//example with workorder report
//{"SockType":34,"selectedRowIds":[355],"ReportId":9,"ClientMeta":{"UserName":"Sockeye SuperUser","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOiIxNjQ2NzgyNTc4IiwiaXNzIjoiYXlhbm92YS5jb20iLCJpZCI6IjEifQ.ad7Acq54JCRGitDWKDJFFnqKkidbdaKaFmj-RA_RG5E","DownloadToken":"NdoU8ca3LG4L39Tj2oi3UReeeM7FLevTgbgopTPhGbA","TimeZoneName":"America/Los_Angeles","LanguageName":"en-US","Hour12":true,"CurrencyName":"USD","DefaultLocale":"en","PDFDate":"3/3/22","PDFTime":"3:38 PM"}}
var reportRequest = new DataListReportRequest();
reportRequest.SockType = ne.SockType;
reportRequest.ReportId = (long)subscription.LinkReportId;
reportRequest.SelectedRowIds = new long[] { ne.ObjectId };
var jwt = Api.Controllers.AuthController.GenRpt(subTranslationId);
//this could be adjusted by culture if we allow user to set a culture but that's getting a bit into the weeds, likely the server default is fine
var pdfDate = new DateTime().ToShortDateString();
var pdfTime = new DateTime().ToShortTimeString();
var h12 = subscription.Hour12 ? "true" : "false";
reportRequest.ClientMeta = JToken.Parse($"{{'UserName':'-','Authorization':'Bearer {jwt}','TimeZoneName':'{subscription.TimeZoneOverride}','LanguageName':'{subscription.LanguageOverride}','Hour12':{h12},'CurrencyName':'{subscription.CurrencyName}','DefaultLocale':'en','PDFDate':'{pdfDate}','PDFTime':'{pdfTime}'}}");
//get port number
var match = System.Text.RegularExpressions.Regex.Match(ServerBootConfig.SOCKEYE_USE_URLS, "[0-9]+");
var API_URL = $"http://127.0.0.1:{match.Value}/api/{SockeyeVersion.CurrentApiVersion}/";
var jobid = await biz.RequestRenderReport(reportRequest, DateTime.UtcNow.AddMinutes(ServerBootConfig.SOCKEYE_REPORT_RENDERING_TIMEOUT), API_URL, "CUSTOMER NOTIFICATION - NO USER");
if (jobid == null)
{
throw new ApplicationException($"Report render job id is null failed to start");
}
else
{
bool done = false;
DateTime bailAfter = DateTime.Now.AddMinutes(ServerBootConfig.SOCKEYE_REPORT_RENDERING_TIMEOUT);
while (!done && DateTime.Now < bailAfter)
{
var status = await JobsBiz.GetJobStatusAsync((Guid)jobid);
switch (status)
{
case JobStatus.Completed:
{
done = true;
//get job logs and parse file name from it
JobOperationsBiz jobopsbiz = new JobOperationsBiz(ct, 1, AuthorizationRoles.BizAdmin);
List<JobOperationsLogInfoItem> log = await jobopsbiz.GetJobLogListAsync((Guid)jobid);
var lastLog = log[log.Count - 1];
var lastLogJ = JObject.Parse(lastLog.StatusText);
var path = (string)lastLogJ["reportfilename"];
var FilePath = FileUtil.GetFullPathForTemporaryFile(path);
var FileName = FileUtil.StringToSafeFileName(await TranslationBiz.GetTranslationStaticAsync(ne.SockType.ToString(), subTranslationId, ct) + $"-{ne.Name}.pdf").ToLowerInvariant();
await m.SendEmailAsync(deliveryAddress, Subject, Body, ServerGlobalOpsSettingsCache.Notify, FilePath, FileName);
break;
}
case JobStatus.Failed:
case JobStatus.Absent:
throw new ApplicationException($"REPORT RENDER JOB {jobid} started but failed");
}
}
if (!done)
throw new TimeoutException("JOB FAILED DUE TO REPORT RENDER TIMEOUT");
}
}
else
await m.SendEmailAsync(deliveryAddress, Subject, Body, ServerGlobalOpsSettingsCache.Notify);
}
}
}
catch (Exception ex)
{
await NotifyEventHelper.AddOpsProblemEvent("SMTP Customer Notification failed", ex);
DeliveryLogItem.Fail = true;
DeliveryLogItem.Error = $"SMTP Notification failed to deliver for this Customer notify event: {ne}, message: {ex.Message}";
log.LogDebug(ex, $"DeliverSMTP Failure delivering Customer notify event: {ne}");
}
finally
{
//remove event no matter what
ct.CustomerNotifyEvent.Remove(ne);
//add delivery log item
await ct.CustomerNotifyDeliveryLog.AddAsync(DeliveryLogItem);
await ct.SaveChangesAsync();
}
}
//===
/////////////////////////////////////////////////////////////////////
}//eoc
}//eons

190
server/generator/CoreJobMetricsSnapshot.cs Normal file
View File

@@ -0,0 +1,190 @@
using System;
using System.Linq;
using System.Diagnostics;
using Microsoft.Extensions.Logging;
using Sockeye.Util;
using Sockeye.Models;
using Microsoft.EntityFrameworkCore;
//using StackExchange.Profiling;
namespace Sockeye.Biz
{
/// <summary>
/// called by Generator to gather server metrics and insert in db
/// </summary>
internal static class CoreJobMetricsSnapshot
{
private static ILogger log = Sockeye.Util.ApplicationLogging.CreateLogger("CoreJobMetricsSnapshot");
private static TimeSpan tsDataRetention = new TimeSpan(365, 0, 0, 0, 0);//one year
private static Process _process = Process.GetCurrentProcess();
private static TimeSpan _oldCPUTime = TimeSpan.Zero;
private static DateTime _lastMMSnapshot = DateTime.UtcNow.Subtract(new TimeSpan(1, 1, 1, 1, 1));//ensure it captures on a fresh boot;
private static DateTime _lastDDSnapshot = DateTime.UtcNow.Subtract(new TimeSpan(1, 1, 1, 1, 1));//ensure it captures on a fresh boot;
private static double _cpu = 0;
#if (DEBUG)
private static TimeSpan tsMMFrequency = new TimeSpan(0, 5, 0);
private static TimeSpan tsDDFrequency = new TimeSpan(24, 0, 0);
#else
private static TimeSpan tsMMFrequency = new TimeSpan(0, 5, 0);
private static TimeSpan tsDDFrequency = new TimeSpan(24, 0, 0);//changed to 12 hours from 24 due to weird issue not gathering like it should diagnosis
#endif
////////////////////////////////////////////////////////////////////////////////////////////////
//
//
public static void DoWork()
{
if (DateUtil.IsAfterDuration(_lastMMSnapshot, tsMMFrequency))
{
/////////////////////////////////////////////
//ONE MINUTE SNAPS
//
log.LogDebug("MM metrics snapshot");
var now = DateTime.UtcNow;
_process.Refresh();
//CPU
var cpuElapsedTime = now.Subtract(_lastMMSnapshot).TotalMilliseconds;
var newCPUTime = _process.TotalProcessorTime;
var elapsedCPU = (newCPUTime - _oldCPUTime).TotalMilliseconds;
_cpu = elapsedCPU * 100 / Environment.ProcessorCount / cpuElapsedTime;
_oldCPUTime = newCPUTime;
//MEMORY
// The memory occupied by objects.
var Allocated = GC.GetTotalMemory(false);//bigint
// The working set includes both shared and private data. The shared data includes the pages that contain all the
// instructions that the process executes, including instructions in the process modules and the system libraries.
var WorkingSet = _process.WorkingSet64;//bigint
// The value returned by this property represents the current size of memory used by the process, in bytes, that
// cannot be shared with other processes.
var PrivateBytes = _process.PrivateMemorySize64;//bigint
//NOTE: CPU percentage is *our* process cpu percentage over timeframe of last captured avg
//So it does *not* show the entire server cpu load, only for RAVEN.
//Overall, server stats need to be captured / viewed independently (digital ocean control panel for example or windows task manager)
var CPU = _cpu;// double precision
//In some cases the first snapshot taken after a reboot
//is a huge number way beyond 100 which fucks up the charts
//likely due to the algorithm above and new values but not worth looking into atm
if (CPU > 100)
{
CPU = 0;
}
using (AyContext ct = ServiceProviderProvider.DBContext)
{
//write to db
MetricMM mm = new MetricMM(Allocated, WorkingSet, PrivateBytes, CPU);
ct.MetricMM.Add(mm);
ct.SaveChanges();
}
_lastMMSnapshot = now;
}
/////////////////////////////////////////////
//ONCE A DAY SNAPS AND CLEANUP
//
if (DateUtil.IsAfterDuration(_lastDDSnapshot, tsDDFrequency))
{
#if (DEBUG)
log.LogInformation($"DD metrics snapshot, _lastDDSnapshot:{_lastDDSnapshot}, tsDDFrequency:{tsDDFrequency}");
#endif
log.LogDebug("DD metrics snapshot");
var now = DateTime.UtcNow;
//FILES ON DISK
var UtilFilesInfo = FileUtil.GetBackupFolderSizeInfo();
var AttachmentFilesInfo = FileUtil.GetAttachmentFolderSizeInfo();
//Available space
long UtilityFilesAvailableSpace = 0;
try
{
UtilityFilesAvailableSpace = FileUtil.BackupFilesDriveAvailableSpace();
}
catch (Exception ex)
{
log.LogError(ex, "Metrics::FileUtil::UtilityFilesDriveAvailableSpace error getting available space");
}
long AttachmentFilesAvailableSpace = 0;
try
{
AttachmentFilesAvailableSpace = FileUtil.AttachmentFilesDriveAvailableSpace();
}
catch (Exception ex)
{
log.LogError(ex, "Metrics::FileUtil::AttachmentFilesDriveAvailableSpace error getting available space");
}
using (AyContext ct = ServiceProviderProvider.DBContext)
{
//DB total size
long DBTotalSize = 0;
using (var command = ct.Database.GetDbConnection().CreateCommand())
{
command.CommandText = "select pg_database_size(current_database());";
ct.Database.OpenConnection();
using (var dr = command.ExecuteReader())
{
if (dr.HasRows)
{
DBTotalSize = dr.Read() ? dr.GetInt64(0) : 0;
}
ct.Database.CloseConnection();
}
}
//write to db
MetricDD dd = new MetricDD()
{
AttachmentFileSize = AttachmentFilesInfo.SizeWithChildren,
AttachmentFileCount = AttachmentFilesInfo.FileCountWithChildren,
AttachmentFilesAvailableSpace = AttachmentFilesAvailableSpace,
UtilityFileSize = UtilFilesInfo.SizeWithChildren,
UtilityFileCount = UtilFilesInfo.FileCountWithChildren,
UtilityFilesAvailableSpace = UtilityFilesAvailableSpace,
DBTotalSize = DBTotalSize
};
ct.MetricDD.Add(dd);
ct.SaveChanges();
}
/////////////////////////////////
//CLEAR OLD ENTRIES
//
DateTime ClearDate = DateTime.UtcNow - tsDataRetention;
using (AyContext ct = ServiceProviderProvider.DBContext)
{
ct.Database.ExecuteSqlInterpolated($"delete from ametricmm where t < {ClearDate.ToUniversalTime()}");
ct.Database.ExecuteSqlInterpolated($"delete from ametricdd where t < {ClearDate.ToUniversalTime()}");
}
_lastDDSnapshot = now;
}
//----
}
/////////////////////////////////////////////////////////////////////
}//eoc
}//eons

411
server/generator/CoreJobNotify.cs Normal file
View File

@@ -0,0 +1,411 @@
using System;
using System.Linq;
using System.Collections.Generic;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Sockeye.Models;
using Sockeye.Util;
namespace Sockeye.Biz
{
/// <summary>
/// Notification processor
/// turn notifyEvent records into inappnotification records for in app viewing and / or deliver smtp notifications seperately
///
/// </summary>
internal static class CoreJobNotify
{
private static bool NotifyIsRunning = false;
private static ILogger log = Sockeye.Util.ApplicationLogging.CreateLogger("CoreJobNotify");
private static DateTime lastRun = DateTime.MinValue;
private static DateTime lastNotifyHealthCheckSentLocal = DateTime.MinValue;
private static TimeSpan TS_24_HOURS = new TimeSpan(24, 0, 0);//used to ensure daily ops happen no more than that
#if (DEBUG)
private static TimeSpan RUN_EVERY_INTERVAL = new TimeSpan(0, 0, 21);//no more frequently than once every 20 seconds
#else
private static TimeSpan RUN_EVERY_INTERVAL = new TimeSpan(0, 1, 1);//no more frequently than once every 1 minute
#endif
////////////////////////////////////////////////////////////////////////////////////////////////
// DoSweep
//
public static async Task DoWorkAsync()
{
log.LogTrace("Checking if Notify should run");
if (NotifyIsRunning)
{
log.LogTrace("Notify is running already exiting this cycle");
return;
}
//This will get triggered roughly every minute, but we don't want to deliver that frequently
if (DateTime.UtcNow - lastRun < RUN_EVERY_INTERVAL)
{
log.LogTrace($"Notify ran less than {RUN_EVERY_INTERVAL} ago, exiting this cycle");
return;
}
try
{
NotifyIsRunning = true;
log.LogDebug("Notify set to RUNNING state and starting now");
//NotifyHealthCheck processing
//Note this deliberately uses LOCAL time in effort to deliver the health check first thing in the morning for workers
//However if server is on UTC already then that's what is used, there is no adjustment
DateTime dtNowLocal = DateTime.Now;
if (dtNowLocal - lastNotifyHealthCheckSentLocal > TS_24_HOURS)
{
//are we in the 7th to 9th hour?
if (dtNowLocal.Hour > 6 && dtNowLocal.Hour < 10)
{
log.LogDebug("Notify health check submitted to subscribers");
await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.NotifyHealthCheck, "OK", "");
lastNotifyHealthCheckSentLocal = dtNowLocal;
}
}
using (AyContext ct = Sockeye.Util.ServiceProviderProvider.DBContext)
{
var events = await ct.NotifyEvent.AsNoTracking().ToListAsync();
log.LogDebug($"Found {events.Count} NotifyEvents to examine for potential delivery");
//iterate and deliver
foreach (var notifyevent in events)
{
//no notifications for inactive users, just delete it as if it was delivered
var UserInfo = await ct.User.AsNoTracking().Where(x => x.Id == notifyevent.UserId).Select(x => new { Active = x.Active, Name = x.Name }).FirstOrDefaultAsync();
if (!UserInfo.Active)
{
log.LogDebug($"Inactive user {UserInfo.Name}, removing notify rather than delivering it: {notifyevent}");
ct.NotifyEvent.Remove(notifyevent);
await ct.SaveChangesAsync();
continue;
}
//Get subscription for delivery
var Subscription = await ct.NotifySubscription.AsNoTracking().FirstOrDefaultAsync(x => x.Id == notifyevent.NotifySubscriptionId);
//NOTE: There is no need to separate out future delivery and immediate delivery because
// All events have an event date, it's either immediate upon creation or it's future
// but not all events have an age value including ones with future event dates,
// and the default agevalue and advancenotice are both zero regardless so the block below works for either future or immediate deliveries
var deliverAfter = notifyevent.EventDate + Subscription.AgeValue - Subscription.AdvanceNotice;
if (deliverAfter < DateTime.UtcNow)
{
//Check "circuit breaker" for notification types that could
//repeat rapidly
//(e.g. pm notification error for a fucked up PM that is attempted every few minutes or a
//system exception for something that pops up every few minutes or a thousand times in a hour etc)
//Don't check for ones that are regular object based
//which can and will properly send out the same notification regularly
//(e.g. workorder status change into out of and back into the same staus)
switch (notifyevent.EventType)
{
case NotifyEventType.BackupStatus:
case NotifyEventType.GeneralNotification:
case NotifyEventType.ServerOperationsProblem:
//case NotifyEventType.PMGenerationFailed:
{
//check if we've just delivered this same thing in the last 12 hours which is the hard limit (case 3917)
var twelvehoursago = DateTime.UtcNow - new TimeSpan(12, 0, 0);
//look for same delivery less than last12hours ago
if (await ct.NotifyDeliveryLog.AnyAsync(z => z.Processed > twelvehoursago && z.NotifySubscriptionId == notifyevent.NotifySubscriptionId && z.ObjectId == notifyevent.ObjectId))
{
log.LogDebug($"Notification event will not be delivered: repetitive (server system event type and delivered at least once in the last 12 hours to this subscriber: {notifyevent})");
ct.NotifyEvent.Remove(notifyevent);
await ct.SaveChangesAsync();
#if (DEBUG)
log.LogInformation($"DeliverInApp event will not be delivered: repetitive (server system event type and delivered at least once in the last 12 hours to this subscriber: {notifyevent})");
#endif
continue;
}
}
break;
}
//Do the delivery, it's kosher
if (Subscription.DeliveryMethod == NotifyDeliveryMethod.App)
await DeliverInApp(notifyevent, Subscription.AgeValue, ct);
else if (Subscription.DeliveryMethod == NotifyDeliveryMethod.SMTP)
await DeliverUserNotificationSMTP(notifyevent, Subscription.AgeValue, Subscription.AdvanceNotice, Subscription.DeliveryAddress, ct);
}
}
}
}
catch (Exception ex)
{
log.LogError(ex, $"Error processing notification event");
DbUtil.HandleIfDatabaseUnavailableTypeException(ex);
}
finally
{
log.LogDebug("Notify is done setting to not running state and tagging lastRun timestamp");
lastRun = DateTime.UtcNow;
NotifyIsRunning = false;
}
}
private static async Task DeliverInApp(NotifyEvent ne, TimeSpan ageValue, AyContext ct)
{
log.LogDebug($"DeliverInApp notify event: {ne}");
//Place in the In-app notification table for user to view
await ct.InAppNotification.AddAsync(
new InAppNotification()
{
UserId = ne.UserId,
SockType = ne.SockType,
ObjectId = ne.ObjectId,
EventType = ne.EventType,
NotifySubscriptionId = ne.NotifySubscriptionId,
Message = ne.Message,
Name = ne.Name,
AgeValue = ageValue,
DecValue = ne.DecValue
});
ct.NotifyEvent.Remove(ne);
//add delivery log item
await ct.NotifyDeliveryLog.AddAsync(new NotifyDeliveryLog()
{
Processed = DateTime.UtcNow,
ObjectId = ne.ObjectId,
NotifySubscriptionId = ne.NotifySubscriptionId,
Fail = false
});
await ct.SaveChangesAsync();
}
private static async Task DeliverUserNotificationSMTP(NotifyEvent ne, TimeSpan ageValue, TimeSpan advanceNotice, string deliveryAddress, AyContext ct)
{
var DeliveryLogItem = new NotifyDeliveryLog()
{
Processed = DateTime.UtcNow,
ObjectId = ne.ObjectId,
NotifySubscriptionId = ne.NotifySubscriptionId,
Fail = false
};
try
{
log.LogDebug($"DeliverSMTP delivering notify event: {ne}");
if (string.IsNullOrWhiteSpace(deliveryAddress))
{
await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.GeneralNotification, $"No email address is set in subscription to deliver email notification. This event will be removed from the delivery queue as undeliverable: {ne}", "Error", null, ne.UserId);
DeliveryLogItem.Fail = true;
DeliveryLogItem.Error = $"No email address provided for smtp delivery; event: {ne}";
}
else
{
//Email notification requires pre-translated values
List<string> TranslationKeysToFetch = new List<string>();
TranslationKeysToFetch.Add(ne.SockType.ToString());
TranslationKeysToFetch.Add("NotifySubscription");
TranslationKeysToFetch.Add("NotifySubscriptionLinkText");
if (ne.Name == "~SERVER~")
TranslationKeysToFetch.Add("Server");
var EventTypeTranslationKey = "NotifyEvent" + ne.EventType.ToString();
TranslationKeysToFetch.Add(EventTypeTranslationKey);
if (ageValue != TimeSpan.Zero )
{
TranslationKeysToFetch.Add("TimeSpanDays");
TranslationKeysToFetch.Add("TimeSpanHours");
TranslationKeysToFetch.Add("TimeSpanMinutes");
TranslationKeysToFetch.Add("TimeSpanSeconds");
}
// if (ne.EventType == NotifyEventType.CustomerServiceImminent)
// TranslationKeysToFetch.Add("NotifyEventCustomerServiceImminentMessage");
// if (ne.DecValue != 0 && ne.EventType == NotifyEventType.WorkorderTotalExceedsThreshold)
// {
// TranslationKeysToFetch.Add("Total");
// }
//get translations
var transid = await ct.UserOptions.AsNoTracking().Where(x => x.UserId == ne.UserId).Select(x => x.TranslationId).FirstOrDefaultAsync();
var LT = await TranslationBiz.GetSubsetStaticAsync(TranslationKeysToFetch, transid);
var NotifySubscriptionLinkText = LT["NotifySubscriptionLinkText"];
var name = ne.Name;
if (name == "~SERVER~")
name = LT["Server"];
//SockType translation
var SockTypeTranslated = "-";
if (ne.SockType != SockType.NoType)
SockTypeTranslated = $"{LT[ne.SockType.ToString()]}";
//subscription name translation
var SubscriptionTypeName = LT[EventTypeTranslationKey];
//Age relevant notification
string AgeDisplay = "";
if (ageValue != TimeSpan.Zero)
AgeDisplay = $"({Sockeye.Util.DateUtil.FormatTimeSpan(ageValue, LT["TimeSpanDays"], LT["TimeSpanHours"], LT["TimeSpanMinutes"], LT["TimeSpanSeconds"])})\n";
//DecValue
string DecDisplay = "";
// //could be money or integer depending on typ
// if (ne.DecValue != 0)
// {
// if (ne.EventType == NotifyEventType.WorkorderTotalExceedsThreshold)
// DecDisplay = $"{LT["Total"]}: {ne.DecValue.ToString("N2", System.Globalization.CultureInfo.InvariantCulture)}\n";
// else if (ne.EventType == NotifyEventType.UnitMeterReadingMultipleExceeded)
// DecDisplay = $"{System.Convert.ToInt64(ne.DecValue).ToString()}\n";
// }
string subject = "";
IMailer m = Sockeye.Util.ServiceProviderProvider.Mailer;
var body = "";
//Special notification handling
switch (ne.EventType)
{
// case NotifyEventType.CustomerServiceImminent:
// subject = SubscriptionTypeName;
// body = LT["NotifyEventCustomerServiceImminentMessage"].Replace("{0}", Sockeye.Util.DateUtil.FormatTimeSpan(advanceNotice, LT["TimeSpanDays"], LT["TimeSpanHours"], LT["TimeSpanMinutes"], LT["TimeSpanSeconds"]));
// body += $"\n{OpenObjectUrlBuilder(ne.SockType, ne.ObjectId, ne.EventType)}\n";
// break;
default:
subject = $"AY:{SockTypeTranslated}:{name}:{SubscriptionTypeName}";
if (ne.ObjectId != 0 || ne.EventType == NotifyEventType.BackupStatus)
{
body = $"{AgeDisplay}{DecDisplay}{SockTypeTranslated}\n{OpenObjectUrlBuilder(ne.SockType, ne.ObjectId, ne.EventType)}\n";
}
body += ne.Message;
break;
}
//Add link to subscription, all messages have this regardless of content
//http://localhost:8080/open/51/1 //add subscription link, notifysub is object type 51
if (!body.EndsWith('\n'))
body += "\n";
body += $"-----\n{NotifySubscriptionLinkText}\n{OpenSubscriptionUrlBuilder(ne.NotifySubscriptionId)}\n";
if (!ServerGlobalOpsSettingsCache.Notify.SmtpDeliveryActive)
{
await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.GeneralNotification, $"Email notifications are set to OFF at server, unable to send email notification for this event:{ne}", "Error", null, ne.UserId);
log.LogInformation($"** WARNING: SMTP notification is currently set to Active=False; unable to deliver email notification, re-routed to in-app notification instead [UserId={ne.UserId}, Notify subscription={ne.NotifySubscriptionId}]. Change this setting or have users remove email delivery notifications if this is permanent **");
DeliveryLogItem.Fail = true;
DeliveryLogItem.Error = $"Email notifications are set to OFF at server, unable to send email notification for this event: {ne}";
}
else
await m.SendEmailAsync(deliveryAddress, subject, body, ServerGlobalOpsSettingsCache.Notify);
}
}
catch (Exception ex)
{
await NotifyEventHelper.AddOpsProblemEvent("SMTP Notification failed", ex);
await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.GeneralNotification, $"An error prevented delivering the following notification via email. System operator users have been notified:{ne}", "Error", null, ne.UserId);
DeliveryLogItem.Fail = true;
DeliveryLogItem.Error = $"SMTP Notification failed to deliver for this event: {ne}, message: {ex.Message}";
log.LogDebug(ex, $"DeliverSMTP Failure delivering notify event: {ne}");
}
finally
{
//remove event no matter what
ct.NotifyEvent.Remove(ne);
//add delivery log item
await ct.NotifyDeliveryLog.AddAsync(DeliveryLogItem);
await ct.SaveChangesAsync();
}
}
//Called from ops notification settings to test smtp setup by delivering to address of choosing
public static async Task<string> TestSMTPDelivery(string toAddress)
{
IMailer m = Sockeye.Util.ServiceProviderProvider.Mailer;
try
{
await m.SendEmailAsync(toAddress, "Test from Notification system", "This is a test to confirm notification system is working", ServerGlobalOpsSettingsCache.Notify);
return "ok";
}
catch (Exception ex)
{
await NotifyEventHelper.AddOpsProblemEvent("SMTP (TEST) Notification failed", ex);
return ExceptionUtil.ExtractAllExceptionMessages(ex);
}
}
//Open object url
private static string OpenObjectUrlBuilder(SockType aType, long id, NotifyEventType net)
{
var ServerUrl = ServerGlobalOpsSettingsCache.Notify.SockeyeServerURL;
if (string.IsNullOrWhiteSpace(ServerUrl))
{
NotifyEventHelper.AddOpsProblemEvent("Notification system: The OPS Notification setting is empty for Sockeye Server URL. This prevents Notification system from linking events to openable objects.").Wait();
return "OPS ERROR NO SERVER URL CONFIGURED";
}
ServerUrl = ServerUrl.Trim().TrimEnd('/');
//HANDLE ITEMS WITHOUT TYPE OR ID
if (net == NotifyEventType.BackupStatus)
{
return $"{ServerUrl}/ops-backup";
}
//Might not have a type or id in which case nothing directly to open
if (aType == SockType.NoType || id == 0)
{
return ServerUrl;
}
if (NotifyEventType.ObjectDeleted == net)
{
//goto event log for item
// path: "/history/:socktype/:recordid/:userlog?",
return $"{ServerUrl}/history/{(int)aType}/{id}";
}
//default is to open the object in question directly
return $"{ServerUrl}/open/{(int)aType}/{id}";
}
//url to open subscription for editing
private static string OpenSubscriptionUrlBuilder(long id)
{
var ServerUrl = ServerGlobalOpsSettingsCache.Notify.SockeyeServerURL;
if (string.IsNullOrWhiteSpace(ServerUrl))
{
NotifyEventHelper.AddOpsProblemEvent("Notification system: The OPS Notification setting is empty for Sockeye Server URL. This prevents Notification system from linking events to openable objects.").Wait();
return "OPS ERROR NO SERVER URL CONFIGURED";
}
ServerUrl = ServerUrl.Trim().TrimEnd('/');
//default is to open the object in question directly
return $"{ServerUrl}/open/{(int)SockType.NotifySubscription}/{id}";
}
/////////////////////////////////////////////////////////////////////
}//eoc
}//eons

41
server/generator/CoreJobReportRenderEngineProcessCleanup.cs Normal file
View File

@@ -0,0 +1,41 @@
using System;
using Microsoft.Extensions.Logging;
using Sockeye.Util;
using System.Threading.Tasks;
namespace Sockeye.Biz
{
/// <summary>
/// called by Generator to kill report generation processor stuck in limbo (chromium at this time)
/// </summary>
internal static class CoreJobReportRenderEngineProcessCleanup
{
private static ILogger log = Sockeye.Util.ApplicationLogging.CreateLogger("CoreJobReportRenderEngineProcessCleanup");
private static DateTime _lastRun = DateTime.UtcNow;
//SET LOW INTENTIONALLY AS CAN EAT UP A LOT OF RESOURCES QUICKLY IF RUN'S PAST TIME
private static TimeSpan tsRunEvery = new TimeSpan(0, 0, 20);//every twenty seconds run the cleanup task
////////////////////////////////////////////////////////////////////////////////////////////////
//
//
public static async Task DoWork()
{
if (DateUtil.IsAfterDuration(_lastRun, tsRunEvery))
{
log.LogTrace("Checking for expired report jobs");
await Util.ReportRenderManager.KillExpiredRenders(log);
var now = DateTime.UtcNow;
_lastRun = now;
}
}
/////////////////////////////////////////////////////////////////////
}//eoc
}//eons

149
server/generator/CoreJobSweeper.cs Normal file
View File

@@ -0,0 +1,149 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Sockeye.Models;
namespace Sockeye.Biz
{
//#################### NOTE: This also does some license checking tasks, put in here instead of the corejoblicense job deliberately #############
/// <summary>
/// JobSweeper - called by Generator to clean out old jobs that are completed and their logs
///
/// </summary>
internal static class CoreJobSweeper
{
private static ILogger log = Sockeye.Util.ApplicationLogging.CreateLogger("CoreJobSweeper");
private static DateTime lastSweep = DateTime.MinValue;
private static TimeSpan SWEEP_EVERY_INTERVAL = new TimeSpan(0, 10, 10);//every ten minutes roughly
private static TimeSpan SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN = new TimeSpan(14, 0, 0, 0);//14 days
private static TimeSpan FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN = new TimeSpan(14, 0, 0, 0);//14 days (gives people time to notice and look into it)
private static TimeSpan INTERNAL_JOBS_LOGS_DELETE_AFTER_THIS_TIMESPAN = new TimeSpan(14, 0, 0, 0);//14 days
private static TimeSpan RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN = new TimeSpan(24, 0, 0);//24 hours (time running jobs are allowed to sit in "running" state before considered failed)
////////////////////////////////////////////////////////////////////////////////////////////////
// DoSweep
//
public static async Task DoWorkAsync()
{
//This will get triggered roughly every minute, but we don't want to sweep that frequently
if (DateTime.UtcNow - lastSweep < SWEEP_EVERY_INTERVAL)
return;
log.LogDebug("Sweep starting");
using (AyContext ct = Sockeye.Util.ServiceProviderProvider.DBContext)
{
//SWEEP SUCCESSFUL JOBS
//calculate cutoff to delete
DateTime dtDeleteCutoff = DateTime.UtcNow - SUCCEEDED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Completed);
//SWEEP FAILED JOBS
//calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - FAILED_JOBS_DELETE_AFTER_THIS_TIMESPAN;
await sweepAsync(ct, dtDeleteCutoff, JobStatus.Failed);
//KILL STUCK JOBS
//calculate cutoff to delete
DateTime dtRunningDeadline = DateTime.UtcNow - RUNNING_JOBS_BECOME_FAILED_AFTER_THIS_TIMESPAN;
await killStuckJobsAsync(ct, dtRunningDeadline);
//SWEEP INTERNAL JOB LOG
//calculate cutoff to delete
dtDeleteCutoff = DateTime.UtcNow - INTERNAL_JOBS_LOGS_DELETE_AFTER_THIS_TIMESPAN;
await SweepInternalJobsLogsAsync(ct, dtDeleteCutoff);
}
lastSweep = DateTime.UtcNow;
}
private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus)
{
//Get the deleteable succeeded jobs list
var jobs = await ct.OpsJob
.AsNoTracking()
.Where(z => z.Created < dtDeleteCutoff && z.JobStatus == jobStatus)
.OrderBy(z => z.Created)
.ToListAsync();
log.LogDebug($"SweepAsync processing: cutoff={dtDeleteCutoff.ToString()}, for {jobs.Count.ToString()} jobs of status {jobStatus.ToString()}");
foreach (OpsJob j in jobs)
{
try
{
await JobsBiz.RemoveJobAndLogsAsync(j.GId);
}
catch (Exception ex)
{
log.LogError(ex, "sweepAsync exception calling JobsBiz.RemoveJobAndLogsAsync");
//for now just throw it but this needs to be removed when logging added and better handling
throw;
}
}
}
/// <summary>
/// Kill jobs that have been stuck in "running" state for too long
/// </summary>
private static async Task killStuckJobsAsync(AyContext ct, DateTime dtRunningDeadline)
{
//Get the deleteable succeeded jobs list
var jobs = await ct.OpsJob
.AsNoTracking()
.Where(z => z.Created < dtRunningDeadline && z.JobStatus == JobStatus.Running)
.OrderBy(z => z.Created)
.ToListAsync();
log.LogDebug($"killStuckJobsAsync processing: cutoff={dtRunningDeadline.ToString()}, for {jobs.Count.ToString()} jobs of status {JobStatus.Running.ToString()}");
foreach (OpsJob j in jobs)
{
//OPSMETRIC
await JobsBiz.LogJobAsync(j.GId, "LT:JobFailed LT:TimedOut");
log.LogError($"Job found job stuck in running status and set to failed: deadline={dtRunningDeadline.ToString()}, jobId={j.GId.ToString()}, jobname={j.Name}, jobtype={j.JobType.ToString()}, jobAType={j.SockType.ToString()}, jobObjectId={j.ObjectId.ToString()}");
await JobsBiz.UpdateJobStatusAsync(j.GId, JobStatus.Failed);
}
}
private static async Task SweepInternalJobsLogsAsync(AyContext ct, DateTime dtDeleteCutoff)
{
//Get the deleteable list (this is for reporting, could easily just do it in one go)
var logs = await ct.OpsJobLog
.AsNoTracking()
.Where(z => z.Created < dtDeleteCutoff)
.OrderBy(z => z.Created)
.ToListAsync();
log.LogDebug($"SweepInternalJobsLogsAsync processing: cutoff={dtDeleteCutoff.ToString()}, for {logs.Count.ToString()} log entries");
foreach (OpsJobLog l in logs)
{
try
{
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjoblog where gid = {l.GId}");
}
catch (Exception ex)
{
log.LogError(ex, "SweepInternalJobsLogsAsync exception removed old log entries");
throw;
}
}
}
/////////////////////////////////////////////////////////////////////
}//eoc
}//eons

56
server/generator/CoreJobTempFolderCleanup.cs Normal file
View File

@@ -0,0 +1,56 @@
using System;
using Microsoft.Extensions.Logging;
using Sockeye.Util;
namespace Sockeye.Biz
{
/// <summary>
/// called by Generator to keep temp folder squeaky clean
/// </summary>
internal static class CoreJobTempFolderCleanup
{
private static ILogger log = Sockeye.Util.ApplicationLogging.CreateLogger("CoreJobTempFolderCleanup");
private static DateTime _lastRun = DateTime.UtcNow;
#if (DEBUG)
private static TimeSpan RUN_EVERY_INTERVAL = new TimeSpan(0, 0, 21);//no more frequently than once every 20 seconds
#else
private static TimeSpan RUN_EVERY_INTERVAL = new TimeSpan(0, 5, 2);//no more frequently than once every 5 minutes
#endif
// private static TimeSpan tsRunEvery = new TimeSpan(0, 5, 2);//every this minutes run the cleanup task
//erase any files found to be older than 15 minutes (which coincides with max report rendering timeout)
#if (DEBUG)
private static TimeSpan DELETE_IF_OLDER_THAN = new TimeSpan(0, 2, 1);//2 minutes max
#else
private static TimeSpan DELETE_IF_OLDER_THAN = new TimeSpan(0, 15, 1);
#endif
////////////////////////////////////////////////////////////////////////////////////////////////
//
//
public static void DoWork()
{
if (DateUtil.IsAfterDuration(_lastRun, RUN_EVERY_INTERVAL))
{
log.LogDebug("Temp cleanup now");
FileUtil.CleanTemporaryFilesFolder(DELETE_IF_OLDER_THAN);
var now = DateTime.UtcNow;
_lastRun = now;
}
}
/////////////////////////////////////////////////////////////////////
}//eoc
}//eons

64
server/generator/CoreNotificationSweeper.cs Normal file
View File

@@ -0,0 +1,64 @@
using System;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Sockeye.Models;
namespace Sockeye.Biz
{
/// <summary>
/// Clear out old data no longer required for notification system
/// </summary>
internal static class CoreNotificationSweeper
{
private static ILogger log = Sockeye.Util.ApplicationLogging.CreateLogger("CoreNotificationSweeper");
private static DateTime lastSweep = DateTime.MinValue;
private static TimeSpan DELETE_AFTER_AGE = new TimeSpan(90, 0, 0, 0);//## WARNING CoreJobPMInventoryCheck EXPECTS THIS TO BE a pretty big value 90 days right now
private static TimeSpan SWEEP_EVERY_INTERVAL = new TimeSpan(8, 0, 0);//once every 8 hours, three times a day
////////////////////////////////////////////////////////////////////////////////////////////////
// DoSweep
//
public static async Task DoWorkAsync()
{
//This will get triggered roughly every minute, but we don't want to sweep that frequently
if (DateTime.UtcNow - lastSweep < SWEEP_EVERY_INTERVAL)
return;
DateTime dtDeleteCutoff = DateTime.UtcNow - DELETE_AFTER_AGE;
DateTime dtPastEventCutoff = DateTime.UtcNow - SWEEP_EVERY_INTERVAL;
log.LogDebug("Sweep starting");
using (AyContext ct = Sockeye.Util.ServiceProviderProvider.DBContext)
{
//Notification (in-App notifications table) - deletes all APP notifications older than 90 days (if they want to keep it then can turn it into a reminder or SAVE it somehow)
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from ainappnotification where created < {dtDeleteCutoff}");
//NotifyEvent - deletes any notifyevent with no event date created more than 90 days ago
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from anotifyevent where eventdate is null and created < {dtDeleteCutoff}");
//NotifyEvent - If has event date and it's in the past by more than an 8 hours (or whatever sweep interval is, to allow for items about to be delivered momentarily as delivery is on briefer cycle than sweep)
//then deletes it if created more than 90 days ago (pretty sure there are no back dated events, once it's passed it's past)
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from anotifyevent where eventdate < {dtPastEventCutoff} and created < {dtDeleteCutoff}");
//NotifyDeliveryLog - deletes all log items older than 90 days (NOTE: this log is also used to identify and prevent high frequency repetitive dupes)
//Note: also has bearing on CoreJobPMInventoryCheck which uses notifydeliverylog to check if pm inventory notify has been done it's one time already
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from anotifydeliverylog where processed < {dtDeleteCutoff}");
//CustomerNotifyDeliveryLog - deletes all log items older than 90 days
await ct.Database.ExecuteSqlInterpolatedAsync($"delete from acustomernotifydeliverylog where processed < {dtDeleteCutoff}");
}
lastSweep = DateTime.UtcNow;
}
/////////////////////////////////////////////////////////////////////
}//eoc
}//eons

81
server/generator/Generate.cs Normal file
View File

@@ -0,0 +1,81 @@
using System.Threading;
using System;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Sockeye.Biz;
using Sockeye.Util;
namespace Sockeye.Generator
{
//Implemented from a example here
//https://docs.microsoft.com/en-us/dotnet/standard/microservices-architecture/multi-container-microservice-net-applications/background-tasks-with-ihostedservice
/*
LOOKAT: Generator tasks that should happen:
- Periodically erase any temp files written to userfiles root (attachments temp files) that are older than a day
- These files should be normally erased within seconds after uploading and processing into their permanent folder but shit will go wrong
*/
public class GeneratorService : BackgroundService
{
private readonly ILogger<GeneratorService> log;
// private const int MAXIMUM_MS_ALLOWED_FOR_PROCESSING_ALL_JOBS = 1 * 60 * 1000;//1 minutes TEST TEST TEST #####
// #if (DEBUG)
// private const int GENERATE_SECONDS = 1;
// #else
// private const int GENERATE_SECONDS = 20;
// #endif
public GeneratorService(ILogger<GeneratorService> logger)
{
log = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
log.LogInformation($"GeneratorService is starting.");
stoppingToken.Register(() =>
log.LogDebug($" GeneratorService background task is stopping."));
while (!stoppingToken.IsCancellationRequested)
{
if (!ServerGlobalOpsSettingsCache.BOOTING)
{
// log.LogTrace($"GeneratorService running jobs");
//=================================================================
try
{
await JobsBiz.ProcessJobsAsync();
}
catch (Exception ex)
{
log.LogError(ex, "Generate::ProcessJobs result in exception error ");
}
//=================================================================
}
//There *MUST* be a delay here or the server will come to a standstill
//ideally need to move on from this method of generation to something external or more reasonable but
//this does work, we need to get to release, for our projected use load this should be fine and it's
//a documented by MS method to do asp.net core job processing
//https://docs.microsoft.com/en-us/aspnet/core/fundamentals/host/hosted-services?view=aspnetcore-6.0&tabs=visual-studio
await Task.Delay(1000);
}
log.LogInformation($"GeneratorService is stopping");
}
public override Task StopAsync(CancellationToken stoppingToken)
{
log.LogDebug($"GeneratorService StopAsync triggered");
return Task.FromResult(0);
// Run any needed clean-up actions
}
}//eoc
}//eons