From f522dda8f19bd86309fa6b0d0f7bede6a7e3e111 Mon Sep 17 00:00:00 2001 From: John Cardinal Date: Mon, 2 Aug 2021 22:28:30 +0000 Subject: [PATCH] --- .../AyaNova/Controllers/NotifyController.cs | 10 +-- server/AyaNova/generator/CoreJobNotify.cs | 69 +++++++++++++++++-- .../generator/CoreNotificationSweeper.cs | 9 ++- server/AyaNova/models/AyContext.cs | 2 +- server/AyaNova/models/Notification.cs | 7 +- server/AyaNova/models/NotifyDeliveryLog.cs | 38 +++++----- server/AyaNova/models/NotifyEvent.cs | 1 + server/AyaNova/util/AySchema.cs | 7 +- server/AyaNova/util/DbUtil.cs | 2 +- 9 files changed, 104 insertions(+), 41 deletions(-) diff --git a/server/AyaNova/Controllers/NotifyController.cs b/server/AyaNova/Controllers/NotifyController.cs index 6dd0ee53..646cb564 100644 --- a/server/AyaNova/Controllers/NotifyController.cs +++ b/server/AyaNova/Controllers/NotifyController.cs @@ -71,7 +71,7 @@ namespace AyaNova.Api.Controllers if (serverState.IsClosed) return StatusCode(503, new ApiErrorResponse(serverState.ApiErrorCode, null, serverState.Reason)); var UserId = UserIdFromContext.Id(HttpContext.Items); - return Ok(ApiOkResponse.Response(await ct.Notification.CountAsync(z => z.UserId == UserId && z.Fetched == false))); + return Ok(ApiOkResponse.Response(await ct.InAppNotification.CountAsync(z => z.UserId == UserId && z.Fetched == false))); } /// @@ -84,8 +84,8 @@ namespace AyaNova.Api.Controllers if (serverState.IsClosed) return StatusCode(503, new ApiErrorResponse(serverState.ApiErrorCode, null, serverState.Reason)); var UserId = UserIdFromContext.Id(HttpContext.Items); - var ret = await ct.Notification.AsNoTracking().Where(z => z.UserId == UserId).OrderByDescending(z => z.Created).ToListAsync(); - await ct.Database.ExecuteSqlInterpolatedAsync($"update anotification set fetched={true} where userid = {UserId}"); + var ret = await ct.InAppNotification.AsNoTracking().Where(z => z.UserId == UserId).OrderByDescending(z => z.Created).ToListAsync(); + await ct.Database.ExecuteSqlInterpolatedAsync($"update ainappnotification set fetched={true} where userid = {UserId}"); return Ok(ApiOkResponse.Response(ret)); } @@ -103,12 +103,12 @@ namespace AyaNova.Api.Controllers if (!ModelState.IsValid) return BadRequest(new ApiErrorResponse(ModelState)); var UserId = UserIdFromContext.Id(HttpContext.Items); - var n = await ct.Notification.FirstOrDefaultAsync(z => z.Id == id); + var n = await ct.InAppNotification.FirstOrDefaultAsync(z => z.Id == id); if (n == null) return BadRequest(new ApiErrorResponse(ApiErrorCode.NOT_FOUND, "id")); if (n.UserId != UserId) return BadRequest(new ApiErrorResponse(ApiErrorCode.NOT_AUTHORIZED, null, "Can't delete notification for another user")); - ct.Notification.Remove(n); + ct.InAppNotification.Remove(n); await ct.SaveChangesAsync(); return NoContent(); } diff --git a/server/AyaNova/generator/CoreJobNotify.cs b/server/AyaNova/generator/CoreJobNotify.cs index 8a535888..00e40b3f 100644 --- a/server/AyaNova/generator/CoreJobNotify.cs +++ b/server/AyaNova/generator/CoreJobNotify.cs @@ -12,7 +12,7 @@ namespace AyaNova.Biz /// /// Notification processor - /// turn notifyEvent records into notifications for in app and deliver smtp + /// turn notifyEvent records into inappnotification records for in app viewing and / or deliver smtp notifications seperately /// /// internal static class CoreJobNotify @@ -95,6 +95,39 @@ namespace AyaNova.Biz var deliverAfter = notifyevent.EventDate + Subscription.AgeValue - Subscription.AdvanceNotice; if (deliverAfter < DateTime.UtcNow) { + //Check "circuit breaker" for notification types that could + //repeat rapidly + //(e.g. pm notification error for a fucked up PM that is attempted every few minutes or a + //system exception for something that pops up every few minutes or a thousand times in a hour etc) + //Don't check for ones that are regular object based + //which can and will properly send out the same notification regularly + //(e.g. workorder status change into out of and back into the same staus) + switch (notifyevent.EventType) + { + case NotifyEventType.BackupStatus: + case NotifyEventType.GeneralNotification: + case NotifyEventType.ServerOperationsProblem: + case NotifyEventType.PMGenerationFailed: + { + //check if we've just delivered this same thing in the last 12 hours which is the hard limit (case 3917) + var twelvehoursago = DateTime.UtcNow - new TimeSpan(12, 0, 0); + + //look for same delivery less than last12hours ago + if (await ct.NotifyDeliveryLog.AnyAsync(z => z.Processed > twelvehoursago && z.NotifySubscriptionId == notifyevent.NotifySubscriptionId && z.ObjectId == notifyevent.ObjectId)) + { + log.LogTrace($"Notification event will not be delivered: repetitive (server system event type and delivered at least once in the last 12 hours to this subscriber: {notifyevent})"); + ct.NotifyEvent.Remove(notifyevent); + await ct.SaveChangesAsync(); +#if (DEBUG) + log.LogInformation($"DeliverInApp event will not be delivered: repetitive (server system event type and delivered at least once in the last 12 hours to this subscriber: {notifyevent})"); +#endif + continue; + } + } + break; + } + + //Do the delivery, it's kosher if (Subscription.DeliveryMethod == NotifyDeliveryMethod.App) await DeliverInApp(notifyevent, Subscription.AgeValue, ct); else if (Subscription.DeliveryMethod == NotifyDeliveryMethod.SMTP) @@ -120,8 +153,10 @@ namespace AyaNova.Biz private static async Task DeliverInApp(NotifyEvent ne, TimeSpan ageValue, AyContext ct) { log.LogTrace($"DeliverInApp notify event: {ne}"); - await ct.Notification.AddAsync( - new Notification() + + //Place in the In-app notification table for user to view + await ct.InAppNotification.AddAsync( + new InAppNotification() { UserId = ne.UserId, AyaType = ne.AyaType, @@ -133,8 +168,16 @@ namespace AyaNova.Biz AgeValue = ageValue, DecValue = ne.DecValue }); - await ct.SaveChangesAsync(); + ct.NotifyEvent.Remove(ne); + //add delivery log item + await ct.NotifyDeliveryLog.AddAsync(new NotifyDeliveryLog() + { + Processed = DateTime.UtcNow, + ObjectId = ne.ObjectId, + NotifySubscriptionId = ne.NotifySubscriptionId, + Fail = false + }); await ct.SaveChangesAsync(); } @@ -142,12 +185,22 @@ namespace AyaNova.Biz private static async Task DeliverSMTP(NotifyEvent ne, TimeSpan ageValue, TimeSpan advanceNotice, string deliveryAddress, AyContext ct) { + var DeliveryLogItem = new NotifyDeliveryLog() + { + Processed = DateTime.UtcNow, + ObjectId = ne.ObjectId, + NotifySubscriptionId = ne.NotifySubscriptionId, + Fail = false + }; + try { log.LogTrace($"DeliverSMTP delivering notify event: {ne}"); if (string.IsNullOrWhiteSpace(deliveryAddress)) { await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.GeneralNotification, $"No email address is set in subscription to deliver email notification. This event will be removed from the delivery queue as undeliverable: {ne}", "Error", null, ne.UserId); + DeliveryLogItem.Fail = true; + DeliveryLogItem.Error = $"No email address provided for smtp delivery; event: {ne}"; } else { @@ -241,6 +294,8 @@ namespace AyaNova.Biz { await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.GeneralNotification, $"Email notifications are set to OFF at server, unable to send email notification for this event:{ne}", "Error", null, ne.UserId); log.LogInformation($"** WARNING: SMTP notification is currently set to Active=False; unable to deliver email notification, re-routed to in-app notification instead [UserId={ne.UserId}, Notify subscription={ne.NotifySubscriptionId}]. Change this setting or have users remove email delivery notifications if this is permanent **"); + DeliveryLogItem.Fail = true; + DeliveryLogItem.Error = $"Email notifications are set to OFF at server, unable to send email notification for this event: {ne}"; } else await m.SendEmailAsync(deliveryAddress, subject, body, ServerGlobalOpsSettingsCache.Notify); @@ -251,11 +306,17 @@ namespace AyaNova.Biz { await NotifyEventHelper.AddOpsProblemEvent("SMTP Notification failed", ex); await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.GeneralNotification, $"An error prevented delivering the following notification via email. System operator users have been notified:{ne}", "Error", null, ne.UserId); + DeliveryLogItem.Fail = true; + DeliveryLogItem.Error = $"SMTP Notification failed to deliver for this event: {ne}, message: {ex.Message}"; + log.LogTrace(ex, $"DeliverSMTP Failure delivering notify event: {ne}"); } finally { //remove event no matter what ct.NotifyEvent.Remove(ne); + + //add delivery log item + await ct.NotifyDeliveryLog.AddAsync(DeliveryLogItem); await ct.SaveChangesAsync(); } } diff --git a/server/AyaNova/generator/CoreNotificationSweeper.cs b/server/AyaNova/generator/CoreNotificationSweeper.cs index d3e08602..c9966079 100644 --- a/server/AyaNova/generator/CoreNotificationSweeper.cs +++ b/server/AyaNova/generator/CoreNotificationSweeper.cs @@ -8,8 +8,7 @@ namespace AyaNova.Biz { /// - /// Clean up notification system, keep items down to 90 days - /// + /// Clear out old data no longer required for notification system /// internal static class CoreNotificationSweeper { @@ -32,8 +31,8 @@ namespace AyaNova.Biz log.LogTrace("Sweep starting"); using (AyContext ct = AyaNova.Util.ServiceProviderProvider.DBContext) { - //Notification (App notifications table) - deletes all APP notifications older than 90 days (if they want to keep it then can turn it into a reminder or SAVE it somehow) - await ct.Database.ExecuteSqlInterpolatedAsync($"delete from anotification where created < {dtDeleteCutoff}"); + //Notification (in-App notifications table) - deletes all APP notifications older than 90 days (if they want to keep it then can turn it into a reminder or SAVE it somehow) + await ct.Database.ExecuteSqlInterpolatedAsync($"delete from ainappnotification where created < {dtDeleteCutoff}"); //NotifyEvent - deletes any notifyevent with no event date created more than 90 days ago await ct.Database.ExecuteSqlInterpolatedAsync($"delete from anotifyevent where eventdate is null and created < {dtDeleteCutoff}"); @@ -42,7 +41,7 @@ namespace AyaNova.Biz //then deletes it if created more than 90 days ago (pretty sure there are no back dated events, once it's passed it's past) await ct.Database.ExecuteSqlInterpolatedAsync($"delete from anotifyevent where eventdate < {dtPastEventCutoff} and created < {dtDeleteCutoff}"); - //NotifyDeliveryLog - deletes all log items older than 90 days + //NotifyDeliveryLog - deletes all log items older than 90 days (NOTE: this log is also used to identify and prevent high frequency repetitive dupes) await ct.Database.ExecuteSqlInterpolatedAsync($"delete from anotifydeliverylog where processed < {dtDeleteCutoff}"); } diff --git a/server/AyaNova/models/AyContext.cs b/server/AyaNova/models/AyContext.cs index a9b92acf..511819b0 100644 --- a/server/AyaNova/models/AyContext.cs +++ b/server/AyaNova/models/AyContext.cs @@ -43,7 +43,7 @@ namespace AyaNova.Models public virtual DbSet LoanUnit { get; set; } public virtual DbSet NotifySubscription { get; set; } public virtual DbSet NotifyEvent { get; set; } - public virtual DbSet Notification { get; set; } + public virtual DbSet InAppNotification { get; set; } public virtual DbSet NotifyDeliveryLog { get; set; } public virtual DbSet Part { get; set; } public virtual DbSet PartInventory { get; set; } diff --git a/server/AyaNova/models/Notification.cs b/server/AyaNova/models/Notification.cs index e083242f..47b1a3b2 100644 --- a/server/AyaNova/models/Notification.cs +++ b/server/AyaNova/models/Notification.cs @@ -1,15 +1,12 @@ using System; -using System.Collections.Generic; using AyaNova.Biz; using System.ComponentModel.DataAnnotations; using Newtonsoft.Json; namespace AyaNova.Models { - //NOTE: Any non required field (nullable in DB) sb nullable here, i.e. decimal? not decimal, - //otherwise the server will call it an invalid record if the field isn't sent from client - public class Notification + public class InAppNotification { public long Id { get; set; } public uint Concurrency { get; set; } @@ -32,7 +29,7 @@ namespace AyaNova.Models [Required] public bool Fetched { get; set; } - public Notification() + public InAppNotification() { Created = DateTime.UtcNow; Fetched = false; diff --git a/server/AyaNova/models/NotifyDeliveryLog.cs b/server/AyaNova/models/NotifyDeliveryLog.cs index c3385e34..5f0cce68 100644 --- a/server/AyaNova/models/NotifyDeliveryLog.cs +++ b/server/AyaNova/models/NotifyDeliveryLog.cs @@ -6,8 +6,11 @@ using Newtonsoft.Json; namespace AyaNova.Models { - //NOTE: Any non required field (nullable in DB) sb nullable here, i.e. decimal? not decimal, - //otherwise the server will call it an invalid record if the field isn't sent from client + //This model holds the deliveries that have been attempted in the past 90 days (cleaned out by corenotifysweeper) + //it is used for verification / troubleshooting purposes from the OPS log + //and also used as a circuit breaker by the corejobnotify to ensure users are not spammed with identical messages + + public class NotifyDeliveryLog { @@ -16,20 +19,21 @@ namespace AyaNova.Models [Required] public DateTime Processed { get; set; } - public AyaType AyaType { get; set; } + //public AyaType AyaType { get; set; } public long ObjectId { get; set; } - [Required] - public NotifyEventType EventType { get; set; } + // [Required] + //public NotifyEventType EventType { get; set; } [Required] public long NotifySubscriptionId { get; set; } - [Required] - public long IdValue { get; set; } - [Required] - public decimal DecValue { get; set; } - [Required] - public long UserId { get; set; } - [Required] - public NotifyDeliveryMethod DeliveryMethod { get; set; } + + // [Required] + // public long IdValue { get; set; } + //[Required] + // public decimal DecValue { get; set; } + // [Required] + // public long UserId { get; set; } + // [Required] + // public NotifyDeliveryMethod DeliveryMethod { get; set; } [Required] public bool Fail { get; set; } public string Error { get; set; } @@ -38,9 +42,11 @@ namespace AyaNova.Models public NotifyDeliveryLog() { Processed = DateTime.UtcNow; - IdValue = 0; - DecValue = 0; - AyaType = AyaType.NoType; + Fail = false; + + // IdValue = 0; + // DecValue = 0; + // AyaType = AyaType.NoType; ObjectId = 0; } diff --git a/server/AyaNova/models/NotifyEvent.cs b/server/AyaNova/models/NotifyEvent.cs index 62f75227..c7a1e0fe 100644 --- a/server/AyaNova/models/NotifyEvent.cs +++ b/server/AyaNova/models/NotifyEvent.cs @@ -14,6 +14,7 @@ namespace AyaNova.Models //it's the result of an event happening, not the subscription which is seperate and decides who gets what //when an object is modified it may create a NotifyEvent record if anyone subscribes to that event //it will create one of these for every user with that subscription + //which will then be delivered as a notification later when the corejobnotify job runs if it's deliverable public class NotifyEvent { public long Id { get; set; } diff --git a/server/AyaNova/util/AySchema.cs b/server/AyaNova/util/AySchema.cs index 6e2f79cb..26573dc8 100644 --- a/server/AyaNova/util/AySchema.cs +++ b/server/AyaNova/util/AySchema.cs @@ -415,7 +415,7 @@ BEGIN when 47 then return 'LT:GlobalOps'; when 48 then return 'LT:BizMetrics'; when 49 then return 'LT:Backup'; - when 50 then aytable = 'anotification'; + when 50 then aytable = 'ainappnotification'; when 51 then return 'LT:NotifySubscription'; when 52 then aytable = 'areminder'; when 53 then return 'LT:UnitMeterReading'; @@ -1110,13 +1110,12 @@ $BODY$ LANGUAGE PLPGSQL STABLE"); + "userid BIGINT NOT NULL REFERENCES auser (id), eventdate TIMESTAMP NOT NULL, decvalue DECIMAL(38,18) NULL, message TEXT)"); - await ExecQueryAsync("CREATE TABLE anotification (id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, userid BIGINT NOT NULL REFERENCES auser (id), " + await ExecQueryAsync("CREATE TABLE ainappnotification (id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, userid BIGINT NOT NULL REFERENCES auser (id), " + "created TIMESTAMP NOT NULL, ayatype INTEGER NOT NULL, objectid BIGINT NOT NULL, name TEXT NOT NULL, agevalue INTERVAL, eventtype INTEGER NOT NULL, " + "decvalue DECIMAL(38,18) NULL, notifysubscriptionid BIGINT NOT NULL REFERENCES anotifysubscription(id) ON DELETE CASCADE, message TEXT, fetched BOOL NOT NULL)"); await ExecQueryAsync("CREATE TABLE anotifydeliverylog (id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, processed TIMESTAMP NOT NULL, " - + "ayatype INTEGER NOT NULL, objectid BIGINT NOT NULL, eventtype INTEGER NOT NULL, notifysubscriptionid BIGINT NOT NULL, idvalue BIGINT NOT NULL, " - + "decvalue DECIMAL(38,18) NOT NULL, userid BIGINT NOT NULL REFERENCES auser (id), deliverymethod INTEGER NOT NULL, fail BOOL NOT NULL, error TEXT)"); + + "objectid BIGINT NOT NULL, notifysubscriptionid BIGINT NOT NULL, fail BOOL NOT NULL, error TEXT)"); //LOGO await ExecQueryAsync("CREATE TABLE alogo (id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, " diff --git a/server/AyaNova/util/DbUtil.cs b/server/AyaNova/util/DbUtil.cs index dab74313..fd51ab46 100644 --- a/server/AyaNova/util/DbUtil.cs +++ b/server/AyaNova/util/DbUtil.cs @@ -423,7 +423,7 @@ namespace AyaNova.Util await EraseTableAsync("acontract", conn); //----- NOTIFICATION - await EraseTableAsync("anotification", conn); + await EraseTableAsync("ainappnotification", conn); await EraseTableAsync("anotifyevent", conn); await EraseTableAsync("anotifydeliverylog", conn); await EraseTableAsync("anotifysubscription", conn);