admin/sockeye
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Browse Source
This commit is contained in:
John Cardinal
2022-12-16 06:01:23 +00:00
parent 26c2ae5cc9
commit effd96143f
310 changed files with 48715 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

411
server/generator/CoreJobNotify.cs Normal file
View File

@@ -0,0 +1,411 @@
using System;
using System.Linq;
using System.Collections.Generic;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Sockeye.Models;
using Sockeye.Util;
namespace Sockeye.Biz
{
/// <summary>
/// Notification processor
/// turn notifyEvent records into inappnotification records for in app viewing and / or deliver smtp notifications seperately
///
/// </summary>
internal static class CoreJobNotify
{
private static bool NotifyIsRunning = false;
private static ILogger log = Sockeye.Util.ApplicationLogging.CreateLogger("CoreJobNotify");
private static DateTime lastRun = DateTime.MinValue;
private static DateTime lastNotifyHealthCheckSentLocal = DateTime.MinValue;
private static TimeSpan TS_24_HOURS = new TimeSpan(24, 0, 0);//used to ensure daily ops happen no more than that
#if (DEBUG)
private static TimeSpan RUN_EVERY_INTERVAL = new TimeSpan(0, 0, 21);//no more frequently than once every 20 seconds
#else
private static TimeSpan RUN_EVERY_INTERVAL = new TimeSpan(0, 1, 1);//no more frequently than once every 1 minute
#endif
////////////////////////////////////////////////////////////////////////////////////////////////
// DoSweep
//
public static async Task DoWorkAsync()
{
log.LogTrace("Checking if Notify should run");
if (NotifyIsRunning)
{
log.LogTrace("Notify is running already exiting this cycle");
return;
}
//This will get triggered roughly every minute, but we don't want to deliver that frequently
if (DateTime.UtcNow - lastRun < RUN_EVERY_INTERVAL)
{
log.LogTrace($"Notify ran less than {RUN_EVERY_INTERVAL} ago, exiting this cycle");
return;
}
try
{
NotifyIsRunning = true;
log.LogDebug("Notify set to RUNNING state and starting now");
//NotifyHealthCheck processing
//Note this deliberately uses LOCAL time in effort to deliver the health check first thing in the morning for workers
//However if server is on UTC already then that's what is used, there is no adjustment
DateTime dtNowLocal = DateTime.Now;
if (dtNowLocal - lastNotifyHealthCheckSentLocal > TS_24_HOURS)
{
//are we in the 7th to 9th hour?
if (dtNowLocal.Hour > 6 && dtNowLocal.Hour < 10)
{
log.LogDebug("Notify health check submitted to subscribers");
await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.NotifyHealthCheck, "OK", "");
lastNotifyHealthCheckSentLocal = dtNowLocal;
}
}
using (AyContext ct = Sockeye.Util.ServiceProviderProvider.DBContext)
{
var events = await ct.NotifyEvent.AsNoTracking().ToListAsync();
log.LogDebug($"Found {events.Count} NotifyEvents to examine for potential delivery");
//iterate and deliver
foreach (var notifyevent in events)
{
//no notifications for inactive users, just delete it as if it was delivered
var UserInfo = await ct.User.AsNoTracking().Where(x => x.Id == notifyevent.UserId).Select(x => new { Active = x.Active, Name = x.Name }).FirstOrDefaultAsync();
if (!UserInfo.Active)
{
log.LogDebug($"Inactive user {UserInfo.Name}, removing notify rather than delivering it: {notifyevent}");
ct.NotifyEvent.Remove(notifyevent);
await ct.SaveChangesAsync();
continue;
}
//Get subscription for delivery
var Subscription = await ct.NotifySubscription.AsNoTracking().FirstOrDefaultAsync(x => x.Id == notifyevent.NotifySubscriptionId);
//NOTE: There is no need to separate out future delivery and immediate delivery because
// All events have an event date, it's either immediate upon creation or it's future
// but not all events have an age value including ones with future event dates,
// and the default agevalue and advancenotice are both zero regardless so the block below works for either future or immediate deliveries
var deliverAfter = notifyevent.EventDate + Subscription.AgeValue - Subscription.AdvanceNotice;
if (deliverAfter < DateTime.UtcNow)
{
//Check "circuit breaker" for notification types that could
//repeat rapidly
//(e.g. pm notification error for a fucked up PM that is attempted every few minutes or a
//system exception for something that pops up every few minutes or a thousand times in a hour etc)
//Don't check for ones that are regular object based
//which can and will properly send out the same notification regularly
//(e.g. workorder status change into out of and back into the same staus)
switch (notifyevent.EventType)
{
case NotifyEventType.BackupStatus:
case NotifyEventType.GeneralNotification:
case NotifyEventType.ServerOperationsProblem:
//case NotifyEventType.PMGenerationFailed:
{
//check if we've just delivered this same thing in the last 12 hours which is the hard limit (case 3917)
var twelvehoursago = DateTime.UtcNow - new TimeSpan(12, 0, 0);
//look for same delivery less than last12hours ago
if (await ct.NotifyDeliveryLog.AnyAsync(z => z.Processed > twelvehoursago && z.NotifySubscriptionId == notifyevent.NotifySubscriptionId && z.ObjectId == notifyevent.ObjectId))
{
log.LogDebug($"Notification event will not be delivered: repetitive (server system event type and delivered at least once in the last 12 hours to this subscriber: {notifyevent})");
ct.NotifyEvent.Remove(notifyevent);
await ct.SaveChangesAsync();
#if (DEBUG)
log.LogInformation($"DeliverInApp event will not be delivered: repetitive (server system event type and delivered at least once in the last 12 hours to this subscriber: {notifyevent})");
#endif
continue;
}
}
break;
}
//Do the delivery, it's kosher
if (Subscription.DeliveryMethod == NotifyDeliveryMethod.App)
await DeliverInApp(notifyevent, Subscription.AgeValue, ct);
else if (Subscription.DeliveryMethod == NotifyDeliveryMethod.SMTP)
await DeliverUserNotificationSMTP(notifyevent, Subscription.AgeValue, Subscription.AdvanceNotice, Subscription.DeliveryAddress, ct);
}
}
}
}
catch (Exception ex)
{
log.LogError(ex, $"Error processing notification event");
DbUtil.HandleIfDatabaseUnavailableTypeException(ex);
}
finally
{
log.LogDebug("Notify is done setting to not running state and tagging lastRun timestamp");
lastRun = DateTime.UtcNow;
NotifyIsRunning = false;
}
}
private static async Task DeliverInApp(NotifyEvent ne, TimeSpan ageValue, AyContext ct)
{
log.LogDebug($"DeliverInApp notify event: {ne}");
//Place in the In-app notification table for user to view
await ct.InAppNotification.AddAsync(
new InAppNotification()
{
UserId = ne.UserId,
SockType = ne.SockType,
ObjectId = ne.ObjectId,
EventType = ne.EventType,
NotifySubscriptionId = ne.NotifySubscriptionId,
Message = ne.Message,
Name = ne.Name,
AgeValue = ageValue,
DecValue = ne.DecValue
});
ct.NotifyEvent.Remove(ne);
//add delivery log item
await ct.NotifyDeliveryLog.AddAsync(new NotifyDeliveryLog()
{
Processed = DateTime.UtcNow,
ObjectId = ne.ObjectId,
NotifySubscriptionId = ne.NotifySubscriptionId,
Fail = false
});
await ct.SaveChangesAsync();
}
private static async Task DeliverUserNotificationSMTP(NotifyEvent ne, TimeSpan ageValue, TimeSpan advanceNotice, string deliveryAddress, AyContext ct)
{
var DeliveryLogItem = new NotifyDeliveryLog()
{
Processed = DateTime.UtcNow,
ObjectId = ne.ObjectId,
NotifySubscriptionId = ne.NotifySubscriptionId,
Fail = false
};
try
{
log.LogDebug($"DeliverSMTP delivering notify event: {ne}");
if (string.IsNullOrWhiteSpace(deliveryAddress))
{
await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.GeneralNotification, $"No email address is set in subscription to deliver email notification. This event will be removed from the delivery queue as undeliverable: {ne}", "Error", null, ne.UserId);
DeliveryLogItem.Fail = true;
DeliveryLogItem.Error = $"No email address provided for smtp delivery; event: {ne}";
}
else
{
//Email notification requires pre-translated values
List<string> TranslationKeysToFetch = new List<string>();
TranslationKeysToFetch.Add(ne.SockType.ToString());
TranslationKeysToFetch.Add("NotifySubscription");
TranslationKeysToFetch.Add("NotifySubscriptionLinkText");
if (ne.Name == "~SERVER~")
TranslationKeysToFetch.Add("Server");
var EventTypeTranslationKey = "NotifyEvent" + ne.EventType.ToString();
TranslationKeysToFetch.Add(EventTypeTranslationKey);
if (ageValue != TimeSpan.Zero )
{
TranslationKeysToFetch.Add("TimeSpanDays");
TranslationKeysToFetch.Add("TimeSpanHours");
TranslationKeysToFetch.Add("TimeSpanMinutes");
TranslationKeysToFetch.Add("TimeSpanSeconds");
}
// if (ne.EventType == NotifyEventType.CustomerServiceImminent)
// TranslationKeysToFetch.Add("NotifyEventCustomerServiceImminentMessage");
// if (ne.DecValue != 0 && ne.EventType == NotifyEventType.WorkorderTotalExceedsThreshold)
// {
// TranslationKeysToFetch.Add("Total");
// }
//get translations
var transid = await ct.UserOptions.AsNoTracking().Where(x => x.UserId == ne.UserId).Select(x => x.TranslationId).FirstOrDefaultAsync();
var LT = await TranslationBiz.GetSubsetStaticAsync(TranslationKeysToFetch, transid);
var NotifySubscriptionLinkText = LT["NotifySubscriptionLinkText"];
var name = ne.Name;
if (name == "~SERVER~")
name = LT["Server"];
//SockType translation
var SockTypeTranslated = "-";
if (ne.SockType != SockType.NoType)
SockTypeTranslated = $"{LT[ne.SockType.ToString()]}";
//subscription name translation
var SubscriptionTypeName = LT[EventTypeTranslationKey];
//Age relevant notification
string AgeDisplay = "";
if (ageValue != TimeSpan.Zero)
AgeDisplay = $"({Sockeye.Util.DateUtil.FormatTimeSpan(ageValue, LT["TimeSpanDays"], LT["TimeSpanHours"], LT["TimeSpanMinutes"], LT["TimeSpanSeconds"])})\n";
//DecValue
string DecDisplay = "";
// //could be money or integer depending on typ
// if (ne.DecValue != 0)
// {
// if (ne.EventType == NotifyEventType.WorkorderTotalExceedsThreshold)
// DecDisplay = $"{LT["Total"]}: {ne.DecValue.ToString("N2", System.Globalization.CultureInfo.InvariantCulture)}\n";
// else if (ne.EventType == NotifyEventType.UnitMeterReadingMultipleExceeded)
// DecDisplay = $"{System.Convert.ToInt64(ne.DecValue).ToString()}\n";
// }
string subject = "";
IMailer m = Sockeye.Util.ServiceProviderProvider.Mailer;
var body = "";
//Special notification handling
switch (ne.EventType)
{
// case NotifyEventType.CustomerServiceImminent:
// subject = SubscriptionTypeName;
// body = LT["NotifyEventCustomerServiceImminentMessage"].Replace("{0}", Sockeye.Util.DateUtil.FormatTimeSpan(advanceNotice, LT["TimeSpanDays"], LT["TimeSpanHours"], LT["TimeSpanMinutes"], LT["TimeSpanSeconds"]));
// body += $"\n{OpenObjectUrlBuilder(ne.SockType, ne.ObjectId, ne.EventType)}\n";
// break;
default:
subject = $"AY:{SockTypeTranslated}:{name}:{SubscriptionTypeName}";
if (ne.ObjectId != 0 || ne.EventType == NotifyEventType.BackupStatus)
{
body = $"{AgeDisplay}{DecDisplay}{SockTypeTranslated}\n{OpenObjectUrlBuilder(ne.SockType, ne.ObjectId, ne.EventType)}\n";
}
body += ne.Message;
break;
}
//Add link to subscription, all messages have this regardless of content
//http://localhost:8080/open/51/1 //add subscription link, notifysub is object type 51
if (!body.EndsWith('\n'))
body += "\n";
body += $"-----\n{NotifySubscriptionLinkText}\n{OpenSubscriptionUrlBuilder(ne.NotifySubscriptionId)}\n";
if (!ServerGlobalOpsSettingsCache.Notify.SmtpDeliveryActive)
{
await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.GeneralNotification, $"Email notifications are set to OFF at server, unable to send email notification for this event:{ne}", "Error", null, ne.UserId);
log.LogInformation($"** WARNING: SMTP notification is currently set to Active=False; unable to deliver email notification, re-routed to in-app notification instead [UserId={ne.UserId}, Notify subscription={ne.NotifySubscriptionId}]. Change this setting or have users remove email delivery notifications if this is permanent **");
DeliveryLogItem.Fail = true;
DeliveryLogItem.Error = $"Email notifications are set to OFF at server, unable to send email notification for this event: {ne}";
}
else
await m.SendEmailAsync(deliveryAddress, subject, body, ServerGlobalOpsSettingsCache.Notify);
}
}
catch (Exception ex)
{
await NotifyEventHelper.AddOpsProblemEvent("SMTP Notification failed", ex);
await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.GeneralNotification, $"An error prevented delivering the following notification via email. System operator users have been notified:{ne}", "Error", null, ne.UserId);
DeliveryLogItem.Fail = true;
DeliveryLogItem.Error = $"SMTP Notification failed to deliver for this event: {ne}, message: {ex.Message}";
log.LogDebug(ex, $"DeliverSMTP Failure delivering notify event: {ne}");
}
finally
{
//remove event no matter what
ct.NotifyEvent.Remove(ne);
//add delivery log item
await ct.NotifyDeliveryLog.AddAsync(DeliveryLogItem);
await ct.SaveChangesAsync();
}
}
//Called from ops notification settings to test smtp setup by delivering to address of choosing
public static async Task<string> TestSMTPDelivery(string toAddress)
{
IMailer m = Sockeye.Util.ServiceProviderProvider.Mailer;
try
{
await m.SendEmailAsync(toAddress, "Test from Notification system", "This is a test to confirm notification system is working", ServerGlobalOpsSettingsCache.Notify);
return "ok";
}
catch (Exception ex)
{
await NotifyEventHelper.AddOpsProblemEvent("SMTP (TEST) Notification failed", ex);
return ExceptionUtil.ExtractAllExceptionMessages(ex);
}
}
//Open object url
private static string OpenObjectUrlBuilder(SockType aType, long id, NotifyEventType net)
{
var ServerUrl = ServerGlobalOpsSettingsCache.Notify.SockeyeServerURL;
if (string.IsNullOrWhiteSpace(ServerUrl))
{
NotifyEventHelper.AddOpsProblemEvent("Notification system: The OPS Notification setting is empty for Sockeye Server URL. This prevents Notification system from linking events to openable objects.").Wait();
return "OPS ERROR NO SERVER URL CONFIGURED";
}
ServerUrl = ServerUrl.Trim().TrimEnd('/');
//HANDLE ITEMS WITHOUT TYPE OR ID
if (net == NotifyEventType.BackupStatus)
{
return $"{ServerUrl}/ops-backup";
}
//Might not have a type or id in which case nothing directly to open
if (aType == SockType.NoType || id == 0)
{
return ServerUrl;
}
if (NotifyEventType.ObjectDeleted == net)
{
//goto event log for item
// path: "/history/:socktype/:recordid/:userlog?",
return $"{ServerUrl}/history/{(int)aType}/{id}";
}
//default is to open the object in question directly
return $"{ServerUrl}/open/{(int)aType}/{id}";
}
//url to open subscription for editing
private static string OpenSubscriptionUrlBuilder(long id)
{
var ServerUrl = ServerGlobalOpsSettingsCache.Notify.SockeyeServerURL;
if (string.IsNullOrWhiteSpace(ServerUrl))
{
NotifyEventHelper.AddOpsProblemEvent("Notification system: The OPS Notification setting is empty for Sockeye Server URL. This prevents Notification system from linking events to openable objects.").Wait();
return "OPS ERROR NO SERVER URL CONFIGURED";
}
ServerUrl = ServerUrl.Trim().TrimEnd('/');
//default is to open the object in question directly
return $"{ServerUrl}/open/{(int)SockType.NotifySubscription}/{id}";
}
/////////////////////////////////////////////////////////////////////
}//eoc
}//eons