using System; using System.Linq; using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using AyaNova.Models; using AyaNova.Util; namespace AyaNova.Biz { /// /// Notification processor /// turn notifyEvent records into inappnotification records for in app viewing and / or deliver smtp notifications seperately /// /// internal static class CoreJobNotify { private static bool NotifyIsRunning = false; private static ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("CoreJobNotify"); private static DateTime lastRun = DateTime.MinValue; private static DateTime lastNotifyHealthCheckSentLocal = DateTime.MinValue; private static TimeSpan TS_24_HOURS = new TimeSpan(24, 0, 0);//used to ensure daily ops happen no more than that #if (DEBUG) private static TimeSpan RUN_EVERY_INTERVAL = new TimeSpan(0, 0, 21);//no more frequently than once every 20 seconds #else private static TimeSpan RUN_EVERY_INTERVAL = new TimeSpan(0, 1, 1);//no more frequently than once every 1 minute #endif //////////////////////////////////////////////////////////////////////////////////////////////// // DoSweep // public static async Task DoWorkAsync() { log.LogDebug("Checking if Notify should run"); if (NotifyIsRunning) { log.LogDebug("Notify is running already exiting this cycle"); return; } //This will get triggered roughly every minute, but we don't want to deliver that frequently if (DateTime.UtcNow - lastRun < RUN_EVERY_INTERVAL) { log.LogDebug($"Notify ran less than {RUN_EVERY_INTERVAL} ago, exiting this cycle"); return; } try { NotifyIsRunning = true; log.LogDebug("Notify set to RUNNING state and starting now"); //NotifyHealthCheck processing //Note this deliberately uses LOCAL time in effort to deliver the health check first thing in the morning for workers //However if server is on UTC already then that's what is used, there is no adjustment DateTime dtNowLocal = DateTime.Now; if (dtNowLocal - lastNotifyHealthCheckSentLocal > TS_24_HOURS) { //are we in the 7th to 9th hour? if (dtNowLocal.Hour > 6 && dtNowLocal.Hour < 10) { log.LogDebug("Notify health check submitted to subscribers"); await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.NotifyHealthCheck, "OK", ""); lastNotifyHealthCheckSentLocal = dtNowLocal; } } using (AyContext ct = AyaNova.Util.ServiceProviderProvider.DBContext) { var events = await ct.NotifyEvent.AsNoTracking().ToListAsync(); log.LogDebug($"Found {events.Count} NotifyEvents to examine for potential delivery"); //iterate and deliver foreach (var notifyevent in events) { //no notifications for inactive users, just delete it as if it was delivered var UserInfo = await ct.User.AsNoTracking().Where(x => x.Id == notifyevent.UserId).Select(x => new { Active = x.Active, Name = x.Name }).FirstOrDefaultAsync(); if (!UserInfo.Active) { log.LogDebug($"Inactive user {UserInfo.Name}, removing notify rather than delivering it: {notifyevent}"); ct.NotifyEvent.Remove(notifyevent); await ct.SaveChangesAsync(); continue; } //Get subscription for delivery var Subscription = await ct.NotifySubscription.AsNoTracking().FirstOrDefaultAsync(x => x.Id == notifyevent.NotifySubscriptionId); //NOTE: There is no need to separate out future delivery and immediate delivery because // All events have an event date, it's either immediate upon creation or it's future // but not all events have an age value including ones with future event dates, // and the default agevalue and advancenotice are both zero regardless so the block below works for either future or immediate deliveries var deliverAfter = notifyevent.EventDate + Subscription.AgeValue - Subscription.AdvanceNotice; if (deliverAfter < DateTime.UtcNow) { //Check "circuit breaker" for notification types that could //repeat rapidly //(e.g. pm notification error for a fucked up PM that is attempted every few minutes or a //system exception for something that pops up every few minutes or a thousand times in a hour etc) //Don't check for ones that are regular object based //which can and will properly send out the same notification regularly //(e.g. workorder status change into out of and back into the same staus) switch (notifyevent.EventType) { case NotifyEventType.BackupStatus: case NotifyEventType.GeneralNotification: case NotifyEventType.ServerOperationsProblem: case NotifyEventType.PMGenerationFailed: { //check if we've just delivered this same thing in the last 12 hours which is the hard limit (case 3917) var twelvehoursago = DateTime.UtcNow - new TimeSpan(12, 0, 0); //look for same delivery less than last12hours ago if (await ct.NotifyDeliveryLog.AnyAsync(z => z.Processed > twelvehoursago && z.NotifySubscriptionId == notifyevent.NotifySubscriptionId && z.ObjectId == notifyevent.ObjectId)) { log.LogDebug($"Notification event will not be delivered: repetitive (server system event type and delivered at least once in the last 12 hours to this subscriber: {notifyevent})"); ct.NotifyEvent.Remove(notifyevent); await ct.SaveChangesAsync(); #if (DEBUG) log.LogInformation($"DeliverInApp event will not be delivered: repetitive (server system event type and delivered at least once in the last 12 hours to this subscriber: {notifyevent})"); #endif continue; } } break; } //Do the delivery, it's kosher if (Subscription.DeliveryMethod == NotifyDeliveryMethod.App) await DeliverInApp(notifyevent, Subscription.AgeValue, ct); else if (Subscription.DeliveryMethod == NotifyDeliveryMethod.SMTP) await DeliverSMTP(notifyevent, Subscription.AgeValue, Subscription.AdvanceNotice, Subscription.DeliveryAddress, ct); } } } } catch (Exception ex) { log.LogError(ex, $"Error processing notification event"); } finally { log.LogDebug("Notify is done setting to not running state and tagging lastRun timestamp"); lastRun = DateTime.UtcNow; NotifyIsRunning = false; } } private static async Task DeliverInApp(NotifyEvent ne, TimeSpan ageValue, AyContext ct) { log.LogDebug($"DeliverInApp notify event: {ne}"); //Place in the In-app notification table for user to view await ct.InAppNotification.AddAsync( new InAppNotification() { UserId = ne.UserId, AyaType = ne.AyaType, ObjectId = ne.ObjectId, EventType = ne.EventType, NotifySubscriptionId = ne.NotifySubscriptionId, Message = ne.Message, Name = ne.Name, AgeValue = ageValue, DecValue = ne.DecValue }); ct.NotifyEvent.Remove(ne); //add delivery log item await ct.NotifyDeliveryLog.AddAsync(new NotifyDeliveryLog() { Processed = DateTime.UtcNow, ObjectId = ne.ObjectId, NotifySubscriptionId = ne.NotifySubscriptionId, Fail = false }); await ct.SaveChangesAsync(); } private static async Task DeliverSMTP(NotifyEvent ne, TimeSpan ageValue, TimeSpan advanceNotice, string deliveryAddress, AyContext ct) { var DeliveryLogItem = new NotifyDeliveryLog() { Processed = DateTime.UtcNow, ObjectId = ne.ObjectId, NotifySubscriptionId = ne.NotifySubscriptionId, Fail = false }; try { log.LogDebug($"DeliverSMTP delivering notify event: {ne}"); if (string.IsNullOrWhiteSpace(deliveryAddress)) { await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.GeneralNotification, $"No email address is set in subscription to deliver email notification. This event will be removed from the delivery queue as undeliverable: {ne}", "Error", null, ne.UserId); DeliveryLogItem.Fail = true; DeliveryLogItem.Error = $"No email address provided for smtp delivery; event: {ne}"; } else { //Email notification requires pre-translated values List TranslationKeysToFetch = new List(); TranslationKeysToFetch.Add(ne.AyaType.ToString()); TranslationKeysToFetch.Add("NotifySubscription"); TranslationKeysToFetch.Add("NotifySubscriptionLinkText"); if (ne.Name == "~SERVER~") TranslationKeysToFetch.Add("Server"); var EventTypeTranslationKey = "NotifyEvent" + ne.EventType.ToString(); TranslationKeysToFetch.Add(EventTypeTranslationKey); if (ageValue != TimeSpan.Zero || ne.EventType == NotifyEventType.CustomerServiceImminent) { TranslationKeysToFetch.Add("TimeSpanDays"); TranslationKeysToFetch.Add("TimeSpanHours"); TranslationKeysToFetch.Add("TimeSpanMinutes"); TranslationKeysToFetch.Add("TimeSpanSeconds"); } if (ne.EventType == NotifyEventType.CustomerServiceImminent) TranslationKeysToFetch.Add("NotifyEventCustomerServiceImminentMessage"); if (ne.DecValue != 0 && ne.EventType == NotifyEventType.WorkorderTotalExceedsThreshold) { TranslationKeysToFetch.Add("Total"); } //get translations var transid = await ct.UserOptions.AsNoTracking().Where(x => x.UserId == ne.UserId).Select(x => x.TranslationId).FirstOrDefaultAsync(); var LT = await TranslationBiz.GetSubsetStaticAsync(TranslationKeysToFetch, transid); var NotifySubscriptionLinkText = LT["NotifySubscriptionLinkText"]; var name = ne.Name; if (name == "~SERVER~") name = LT["Server"]; //AyaType translation var AyaTypeTranslated = "-"; if (ne.AyaType != AyaType.NoType) AyaTypeTranslated = $"{LT[ne.AyaType.ToString()]}"; //subscription name translation var SubscriptionTypeName = LT[EventTypeTranslationKey]; //Age relevant notification string AgeDisplay = ""; if (ageValue != TimeSpan.Zero) AgeDisplay = $"({AyaNova.Util.DateUtil.FormatTimeSpan(ageValue, LT["TimeSpanDays"], LT["TimeSpanHours"], LT["TimeSpanMinutes"], LT["TimeSpanSeconds"])})\n"; //DecValue string DecDisplay = ""; //could be money or integer depending on typ if (ne.DecValue != 0) { if (ne.EventType == NotifyEventType.WorkorderTotalExceedsThreshold) DecDisplay = $"{LT["Total"]}: {ne.DecValue.ToString("N2", System.Globalization.CultureInfo.InvariantCulture)}\n"; else if (ne.EventType == NotifyEventType.UnitMeterReadingMultipleExceeded) DecDisplay = $"{System.Convert.ToInt64(ne.DecValue).ToString()}\n"; } string subject = ""; IMailer m = AyaNova.Util.ServiceProviderProvider.Mailer; var body = ""; //Special notification handling switch (ne.EventType) { case NotifyEventType.CustomerServiceImminent: subject = SubscriptionTypeName; body = LT["NotifyEventCustomerServiceImminentMessage"].Replace("{0}", AyaNova.Util.DateUtil.FormatTimeSpan(advanceNotice, LT["TimeSpanDays"], LT["TimeSpanHours"], LT["TimeSpanMinutes"], LT["TimeSpanSeconds"])); body += $"\n{OpenObjectUrlBuilder(ne.AyaType, ne.ObjectId, ne.EventType)}\n"; break; default: subject = $"AY:{AyaTypeTranslated}:{name}:{SubscriptionTypeName}"; if (ne.ObjectId != 0 || ne.EventType == NotifyEventType.BackupStatus) { body = $"{AgeDisplay}{DecDisplay}{AyaTypeTranslated}\n{OpenObjectUrlBuilder(ne.AyaType, ne.ObjectId, ne.EventType)}\n"; } body += ne.Message; break; } //Add link to subscription, all messages have this regardless of content //http://localhost:8080/open/51/1 //add subscription link, notifysub is object type 51 if (!body.EndsWith('\n')) body += "\n"; body += $"-----\n{NotifySubscriptionLinkText}\n{OpenSubscriptionUrlBuilder(ne.NotifySubscriptionId)}\n"; if (!ServerGlobalOpsSettingsCache.Notify.SmtpDeliveryActive) { await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.GeneralNotification, $"Email notifications are set to OFF at server, unable to send email notification for this event:{ne}", "Error", null, ne.UserId); log.LogInformation($"** WARNING: SMTP notification is currently set to Active=False; unable to deliver email notification, re-routed to in-app notification instead [UserId={ne.UserId}, Notify subscription={ne.NotifySubscriptionId}]. Change this setting or have users remove email delivery notifications if this is permanent **"); DeliveryLogItem.Fail = true; DeliveryLogItem.Error = $"Email notifications are set to OFF at server, unable to send email notification for this event: {ne}"; } else await m.SendEmailAsync(deliveryAddress, subject, body, ServerGlobalOpsSettingsCache.Notify); } } catch (Exception ex) { await NotifyEventHelper.AddOpsProblemEvent("SMTP Notification failed", ex); await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.GeneralNotification, $"An error prevented delivering the following notification via email. System operator users have been notified:{ne}", "Error", null, ne.UserId); DeliveryLogItem.Fail = true; DeliveryLogItem.Error = $"SMTP Notification failed to deliver for this event: {ne}, message: {ex.Message}"; log.LogDebug(ex, $"DeliverSMTP Failure delivering notify event: {ne}"); } finally { //remove event no matter what ct.NotifyEvent.Remove(ne); //add delivery log item await ct.NotifyDeliveryLog.AddAsync(DeliveryLogItem); await ct.SaveChangesAsync(); } } //Called from ops notification settings to test smtp setup by delivering to address of choosing public static async Task TestSMTPDelivery(string toAddress) { //DO TEST DELIVERY HERE USING EXACT SAME SETTINGS AS FOR DeliverSMTP above //todo: abstract out email sending to it's own class maybe or whatever method I choose supports the best //https://jasonwatmore.com/post/2020/07/15/aspnet-core-3-send-emails-via-smtp-with-mailkit //https://medium.com/@ffimnsr/sending-email-using-mailkit-in-asp-net-core-web-api-71b946380442 IMailer m = AyaNova.Util.ServiceProviderProvider.Mailer; try { await m.SendEmailAsync(toAddress, "Test from Notification system", "This is a test to confirm notification system is working", ServerGlobalOpsSettingsCache.Notify); return "ok"; } catch (Exception ex) { await NotifyEventHelper.AddOpsProblemEvent("SMTP (TEST) Notification failed", ex); return ExceptionUtil.ExtractAllExceptionMessages(ex); } } //Open object url //### NOTE: If this is required anywhere else, move it to a central BizUtils class in Biz folder callable from here and there private static string OpenObjectUrlBuilder(AyaType aType, long id, NotifyEventType net) { var ServerUrl = ServerGlobalOpsSettingsCache.Notify.AyaNovaServerURL; if (string.IsNullOrWhiteSpace(ServerUrl)) { NotifyEventHelper.AddOpsProblemEvent("Notification system: The OPS Notification setting is empty for AyaNova Server URL. This prevents Notification system from linking events to openable objects.").Wait(); return "OPS ERROR NO SERVER URL CONFIGURED"; } ServerUrl = ServerUrl.Trim().TrimEnd('/'); //HANDLE ITEMS WITHOUT TYPE OR ID if (net == NotifyEventType.BackupStatus) { return $"{ServerUrl}/ops-backup"; } //Might not have a type or id in which case nothing directly to open if (aType == AyaType.NoType || id == 0) { return ServerUrl; } if (NotifyEventType.ObjectDeleted == net) { //goto event log for item // path: "/history/:ayatype/:recordid/:userlog?", return $"{ServerUrl}/history/{(int)aType}/{id}"; } //default is to open the object in question directly return $"{ServerUrl}/open/{(int)aType}/{id}"; } // //Used to directly open a report at client // private static string OpenReportUrlBuilder(long objectId, long reportId) // { // //CLIENT EXPECTS: open report links to have a query string [CLIENTAPPURL]/viewreport?oid=[objectid]&rid=[reportid] // var ServerUrl = ServerGlobalOpsSettingsCache.Notify.AyaNovaServerURL; // if (string.IsNullOrWhiteSpace(ServerUrl)) // { // NotifyEventHelper.AddOpsProblemEvent("Notification system: The OPS Notification setting is empty for AyaNova Server URL. This prevents Notification system from linking events to openable objects.").Wait(); // return "OPS ERROR NO SERVER URL CONFIGURED"; // } // ServerUrl = ServerUrl.Trim().TrimEnd('/'); // return $"{ServerUrl}/viewreport?oid={objectId}&rid={reportId}"; // } //url to open subscription for editing private static string OpenSubscriptionUrlBuilder(long id) { var ServerUrl = ServerGlobalOpsSettingsCache.Notify.AyaNovaServerURL; if (string.IsNullOrWhiteSpace(ServerUrl)) { NotifyEventHelper.AddOpsProblemEvent("Notification system: The OPS Notification setting is empty for AyaNova Server URL. This prevents Notification system from linking events to openable objects.").Wait(); return "OPS ERROR NO SERVER URL CONFIGURED"; } ServerUrl = ServerUrl.Trim().TrimEnd('/'); //default is to open the object in question directly return $"{ServerUrl}/open/{(int)AyaType.NotifySubscription}/{id}"; } ///////////////////////////////////////////////////////////////////// }//eoc }//eons