server/AyaNova/Controllers/NotifyController.cs

@@ -71,7 +71,7 @@ namespace AyaNova.Api.Controllers
             if (serverState.IsClosed)
                 return StatusCode(503, new ApiErrorResponse(serverState.ApiErrorCode, null, serverState.Reason));
             var UserId = UserIdFromContext.Id(HttpContext.Items);
-            return Ok(ApiOkResponse.Response(await ct.Notification.CountAsync(z => z.UserId == UserId && z.Fetched == false)));
+            return Ok(ApiOkResponse.Response(await ct.InAppNotification.CountAsync(z => z.UserId == UserId && z.Fetched == false)));
         }
 
         /// <summary>
@@ -84,8 +84,8 @@ namespace AyaNova.Api.Controllers
             if (serverState.IsClosed)
                 return StatusCode(503, new ApiErrorResponse(serverState.ApiErrorCode, null, serverState.Reason));
             var UserId = UserIdFromContext.Id(HttpContext.Items);
-            var ret = await ct.Notification.AsNoTracking().Where(z => z.UserId == UserId).OrderByDescending(z => z.Created).ToListAsync();
+            var ret = await ct.InAppNotification.AsNoTracking().Where(z => z.UserId == UserId).OrderByDescending(z => z.Created).ToListAsync();
-            await ct.Database.ExecuteSqlInterpolatedAsync($"update anotification set fetched={true} where userid = {UserId}");
+            await ct.Database.ExecuteSqlInterpolatedAsync($"update ainappnotification set fetched={true} where userid = {UserId}");
             return Ok(ApiOkResponse.Response(ret));
         }
 
@@ -103,12 +103,12 @@ namespace AyaNova.Api.Controllers
             if (!ModelState.IsValid)
                 return BadRequest(new ApiErrorResponse(ModelState));
             var UserId = UserIdFromContext.Id(HttpContext.Items);
-            var n = await ct.Notification.FirstOrDefaultAsync(z => z.Id == id);
+            var n = await ct.InAppNotification.FirstOrDefaultAsync(z => z.Id == id);
             if (n == null)
                 return BadRequest(new ApiErrorResponse(ApiErrorCode.NOT_FOUND, "id"));
             if (n.UserId != UserId)
                 return BadRequest(new ApiErrorResponse(ApiErrorCode.NOT_AUTHORIZED, null, "Can't delete notification for another user"));
-            ct.Notification.Remove(n);
+            ct.InAppNotification.Remove(n);
             await ct.SaveChangesAsync();
             return NoContent();
         }

server/AyaNova/generator/CoreJobNotify.cs

@@ -12,7 +12,7 @@ namespace AyaNova.Biz
 
     /// <summary>
     /// Notification processor
-    /// turn notifyEvent records into notifications for in app and deliver smtp
+    /// turn notifyEvent records into inappnotification records for in app viewing and / or deliver smtp notifications seperately
     /// 
     /// </summary>
     internal static class CoreJobNotify
@@ -95,6 +95,39 @@ namespace AyaNova.Biz
                         var deliverAfter = notifyevent.EventDate + Subscription.AgeValue - Subscription.AdvanceNotice;
                         if (deliverAfter < DateTime.UtcNow)
                         {
+                            //Check "circuit breaker" for notification types that could
+                            //repeat rapidly 
+                            //(e.g. pm notification error for a fucked up PM that is attempted every few minutes or a 
+                            //system exception for something that pops up every few minutes or a thousand times in a hour etc)
+                            //Don't check for ones that are regular object based
+                            //which can and will properly send out the same notification regularly
+                            //(e.g. workorder status change into out of and back into the same staus)          
+                            switch (notifyevent.EventType)
+                            {
+                                case NotifyEventType.BackupStatus:
+                                case NotifyEventType.GeneralNotification:
+                                case NotifyEventType.ServerOperationsProblem:
+                                case NotifyEventType.PMGenerationFailed:
+                                    {
+                                        //check if we've just delivered this same thing in the last 12 hours which is the hard limit (case 3917)
+                                        var twelvehoursago = DateTime.UtcNow - new TimeSpan(12, 0, 0);
+
+                                        //look for same delivery less than last12hours ago
+                                        if (await ct.NotifyDeliveryLog.AnyAsync(z => z.Processed > twelvehoursago && z.NotifySubscriptionId == notifyevent.NotifySubscriptionId && z.ObjectId == notifyevent.ObjectId))
+                                        {
+                                            log.LogTrace($"Notification event will not be delivered: repetitive (server system event type and delivered at least once in the last 12 hours to this subscriber: {notifyevent})");
+                                            ct.NotifyEvent.Remove(notifyevent);
+                                            await ct.SaveChangesAsync();
+#if (DEBUG)
+                                            log.LogInformation($"DeliverInApp event will not be delivered: repetitive (server system event type and delivered at least once in the last 12 hours to this subscriber: {notifyevent})");
+#endif
+                                            continue;
+                                        }
+                                    }
+                                    break;
+                            }
+
+                            //Do the delivery, it's kosher
                             if (Subscription.DeliveryMethod == NotifyDeliveryMethod.App)
                                 await DeliverInApp(notifyevent, Subscription.AgeValue, ct);
                             else if (Subscription.DeliveryMethod == NotifyDeliveryMethod.SMTP)
@@ -120,8 +153,10 @@ namespace AyaNova.Biz
         private static async Task DeliverInApp(NotifyEvent ne, TimeSpan ageValue, AyContext ct)
         {
             log.LogTrace($"DeliverInApp notify event: {ne}");
-            await ct.Notification.AddAsync(
+
-                new Notification()
+            //Place in the In-app notification table for user to view
+            await ct.InAppNotification.AddAsync(
+                new InAppNotification()
                 {
                     UserId = ne.UserId,
                     AyaType = ne.AyaType,
@@ -133,8 +168,16 @@ namespace AyaNova.Biz
                     AgeValue = ageValue,
                     DecValue = ne.DecValue
                 });
-            await ct.SaveChangesAsync();
+
             ct.NotifyEvent.Remove(ne);
+            //add delivery log item
+            await ct.NotifyDeliveryLog.AddAsync(new NotifyDeliveryLog()
+            {
+                Processed = DateTime.UtcNow,
+                ObjectId = ne.ObjectId,
+                NotifySubscriptionId = ne.NotifySubscriptionId,
+                Fail = false
+            });
             await ct.SaveChangesAsync();
         }
 
@@ -142,12 +185,22 @@ namespace AyaNova.Biz
 
         private static async Task DeliverSMTP(NotifyEvent ne, TimeSpan ageValue, TimeSpan advanceNotice, string deliveryAddress, AyContext ct)
         {
+            var DeliveryLogItem = new NotifyDeliveryLog()
+            {
+                Processed = DateTime.UtcNow,
+                ObjectId = ne.ObjectId,
+                NotifySubscriptionId = ne.NotifySubscriptionId,
+                Fail = false
+            };
+
             try
             {
                 log.LogTrace($"DeliverSMTP delivering notify event: {ne}");
                 if (string.IsNullOrWhiteSpace(deliveryAddress))
                 {
                     await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.GeneralNotification, $"No email address is set in subscription to deliver email notification.  This event will be removed from the delivery queue as undeliverable: {ne}", "Error", null, ne.UserId);
+                    DeliveryLogItem.Fail = true;
+                    DeliveryLogItem.Error = $"No email address provided for smtp delivery; event: {ne}";
                 }
                 else
                 {
@@ -241,6 +294,8 @@ namespace AyaNova.Biz
                     {
                         await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.GeneralNotification, $"Email notifications are set to OFF at server, unable to send email notification for this event:{ne}", "Error", null, ne.UserId);
                         log.LogInformation($"** WARNING: SMTP notification is currently set to Active=False; unable to deliver email notification, re-routed to in-app notification instead [UserId={ne.UserId}, Notify subscription={ne.NotifySubscriptionId}].  Change this setting or have users remove email delivery notifications if this is permanent **");
+                        DeliveryLogItem.Fail = true;
+                        DeliveryLogItem.Error = $"Email notifications are set to OFF at server, unable to send email notification for this event: {ne}";
                     }
                     else
                         await m.SendEmailAsync(deliveryAddress, subject, body, ServerGlobalOpsSettingsCache.Notify);
@@ -251,11 +306,17 @@ namespace AyaNova.Biz
             {
                 await NotifyEventHelper.AddOpsProblemEvent("SMTP Notification failed", ex);
                 await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.GeneralNotification, $"An error prevented delivering the following notification via email. System operator users have been notified:{ne}", "Error", null, ne.UserId);
+                DeliveryLogItem.Fail = true;
+                DeliveryLogItem.Error = $"SMTP Notification failed to deliver for this event: {ne}, message: {ex.Message}";
+                log.LogTrace(ex, $"DeliverSMTP Failure delivering notify event: {ne}");
             }
             finally
             {
                 //remove event no matter what
                 ct.NotifyEvent.Remove(ne);
+
+                //add delivery log item
+                await ct.NotifyDeliveryLog.AddAsync(DeliveryLogItem);
                 await ct.SaveChangesAsync();
             }
         }

server/AyaNova/generator/CoreNotificationSweeper.cs

@@ -8,8 +8,7 @@ namespace AyaNova.Biz
 {
 
     /// <summary>
-    /// Clean up notification system, keep items down to 90 days
+    /// Clear out old data no longer required for notification system
-    /// 
     /// </summary>
     internal static class CoreNotificationSweeper
     {
@@ -32,8 +31,8 @@ namespace AyaNova.Biz
             log.LogTrace("Sweep starting");
             using (AyContext ct = AyaNova.Util.ServiceProviderProvider.DBContext)
             {
-                //Notification (App notifications table) - deletes all APP notifications older than 90 days (if they want to keep it then can turn it into a reminder or SAVE it somehow)
+                //Notification (in-App notifications table) - deletes all APP notifications older than 90 days (if they want to keep it then can turn it into a reminder or SAVE it somehow)
-                await ct.Database.ExecuteSqlInterpolatedAsync($"delete from anotification where created < {dtDeleteCutoff}");
+                await ct.Database.ExecuteSqlInterpolatedAsync($"delete from ainappnotification where created < {dtDeleteCutoff}");
 
                 //NotifyEvent - deletes any notifyevent with no event date created more than 90 days ago
                 await ct.Database.ExecuteSqlInterpolatedAsync($"delete from anotifyevent where eventdate is null and created < {dtDeleteCutoff}");
@@ -42,7 +41,7 @@ namespace AyaNova.Biz
                 //then deletes it if created more than 90 days ago (pretty sure there are no back dated events, once it's passed it's past)
                 await ct.Database.ExecuteSqlInterpolatedAsync($"delete from anotifyevent where eventdate < {dtPastEventCutoff} and created < {dtDeleteCutoff}");
 
-                //NotifyDeliveryLog - deletes all log items older than 90 days
+                //NotifyDeliveryLog - deletes all log items older than 90 days (NOTE: this log is also used to identify and prevent high frequency repetitive dupes)
                 await ct.Database.ExecuteSqlInterpolatedAsync($"delete from anotifydeliverylog where processed < {dtDeleteCutoff}");
 
             }

server/AyaNova/models/AyContext.cs

@@ -43,7 +43,7 @@ namespace AyaNova.Models
         public virtual DbSet<LoanUnit> LoanUnit { get; set; }
         public virtual DbSet<NotifySubscription> NotifySubscription { get; set; }
         public virtual DbSet<NotifyEvent> NotifyEvent { get; set; }
-        public virtual DbSet<Notification> Notification { get; set; }
+        public virtual DbSet<InAppNotification> InAppNotification { get; set; }
         public virtual DbSet<NotifyDeliveryLog> NotifyDeliveryLog { get; set; }
         public virtual DbSet<Part> Part { get; set; }
         public virtual DbSet<PartInventory> PartInventory { get; set; }

server/AyaNova/models/Notification.cs

@@ -1,15 +1,12 @@
 using System;
-using System.Collections.Generic;
 using AyaNova.Biz;
 using System.ComponentModel.DataAnnotations;
 using Newtonsoft.Json;
 
 namespace AyaNova.Models
 {
-    //NOTE: Any non required field (nullable in DB) sb nullable here, i.e. decimal? not decimal, 
-    //otherwise the server will call it an invalid record if the field isn't sent from client
 
-    public class Notification
+    public class InAppNotification
     {
         public long Id { get; set; }
         public uint Concurrency { get; set; }
@@ -32,7 +29,7 @@ namespace AyaNova.Models
         [Required]
         public bool Fetched { get; set; }
 
-        public Notification()
+        public InAppNotification()
         {
             Created = DateTime.UtcNow;
             Fetched = false;

server/AyaNova/models/NotifyDeliveryLog.cs

@@ -6,8 +6,11 @@ using Newtonsoft.Json;
 
 namespace AyaNova.Models
 {
-    //NOTE: Any non required field (nullable in DB) sb nullable here, i.e. decimal? not decimal, 
+    //This model holds the deliveries that have been attempted in the past 90 days (cleaned out by corenotifysweeper)
-    //otherwise the server will call it an invalid record if the field isn't sent from client
+    //it is used for verification / troubleshooting purposes from the OPS log
+    //and also used as a circuit breaker by the corejobnotify to ensure users are not spammed with identical messages
+
+
 
     public class NotifyDeliveryLog
     {
@@ -16,20 +19,21 @@ namespace AyaNova.Models
 
         [Required]
         public DateTime Processed { get; set; }
-        public AyaType AyaType { get; set; }
+        //public AyaType AyaType { get; set; }
         public long ObjectId { get; set; }
-        [Required]
+        // [Required]
-        public NotifyEventType EventType { get; set; }
+        //public NotifyEventType EventType { get; set; }
         [Required]
         public long NotifySubscriptionId { get; set; }
-        [Required]
+
-        public long IdValue { get; set; }
+        // [Required]
-        [Required]
+        // public long IdValue { get; set; }
-        public decimal DecValue { get; set; }
+        //[Required]
-        [Required]
+        // public decimal DecValue { get; set; }
-        public long UserId { get; set; }
+        // [Required]
-        [Required]
+        //  public long UserId { get; set; }
-        public NotifyDeliveryMethod DeliveryMethod { get; set; }
+        //  [Required]
+        //  public NotifyDeliveryMethod DeliveryMethod { get; set; }
         [Required]
         public bool Fail { get; set; }
         public string Error { get; set; }
@@ -38,9 +42,11 @@ namespace AyaNova.Models
         public NotifyDeliveryLog()
         {
             Processed = DateTime.UtcNow;
-            IdValue = 0;
+            Fail = false;
-            DecValue = 0;
+
-            AyaType = AyaType.NoType;
+            // IdValue = 0;
+            // DecValue = 0;
+            // AyaType = AyaType.NoType;
             ObjectId = 0;
         }
 

server/AyaNova/models/NotifyEvent.cs

@@ -14,6 +14,7 @@ namespace AyaNova.Models
     //it's the result of an event happening, not the subscription which is seperate and decides who gets what
     //when an object is modified it may create a NotifyEvent record if anyone subscribes to that event 
     //it will create one of these for every user with that subscription
+    //which will then be delivered as a notification later when the corejobnotify job runs if it's deliverable
     public class NotifyEvent
     {
         public long Id { get; set; }

server/AyaNova/util/AySchema.cs

@@ -415,7 +415,7 @@ BEGIN
 		when 47 then return 'LT:GlobalOps';
 		when 48 then return 'LT:BizMetrics';
 		when 49 then return 'LT:Backup';
-		when 50 then aytable = 'anotification';
+		when 50 then aytable = 'ainappnotification';
 		when 51 then return 'LT:NotifySubscription';
         when 52 then aytable = 'areminder';
         when 53 then return 'LT:UnitMeterReading';
@@ -1110,13 +1110,12 @@ $BODY$ LANGUAGE PLPGSQL STABLE");
                     + "userid BIGINT NOT NULL REFERENCES auser (id), eventdate TIMESTAMP NOT NULL, decvalue DECIMAL(38,18) NULL, message TEXT)");
 
 
-                await ExecQueryAsync("CREATE TABLE anotification (id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, userid BIGINT NOT NULL REFERENCES auser (id), "
+                await ExecQueryAsync("CREATE TABLE ainappnotification (id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, userid BIGINT NOT NULL REFERENCES auser (id), "
                     + "created TIMESTAMP NOT NULL, ayatype INTEGER NOT NULL, objectid BIGINT NOT NULL, name TEXT NOT NULL, agevalue INTERVAL, eventtype INTEGER NOT NULL, "
                     + "decvalue DECIMAL(38,18) NULL, notifysubscriptionid BIGINT NOT NULL REFERENCES anotifysubscription(id) ON DELETE CASCADE, message TEXT, fetched BOOL NOT NULL)");
 
                 await ExecQueryAsync("CREATE TABLE anotifydeliverylog (id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, processed TIMESTAMP NOT NULL, "
-                    + "ayatype INTEGER NOT NULL, objectid BIGINT NOT NULL, eventtype INTEGER NOT NULL, notifysubscriptionid BIGINT NOT NULL, idvalue BIGINT NOT NULL, "
+                    + "objectid BIGINT NOT NULL,  notifysubscriptionid BIGINT NOT NULL, fail BOOL NOT NULL, error TEXT)");
-                    + "decvalue DECIMAL(38,18) NOT NULL, userid BIGINT NOT NULL REFERENCES auser (id), deliverymethod INTEGER NOT NULL, fail BOOL NOT NULL, error TEXT)");
 
                 //LOGO
                 await ExecQueryAsync("CREATE TABLE alogo (id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, "

server/AyaNova/util/DbUtil.cs

@@ -423,7 +423,7 @@ namespace AyaNova.Util
                 await EraseTableAsync("acontract", conn);
 
                 //----- NOTIFICATION
-                await EraseTableAsync("anotification", conn);
+                await EraseTableAsync("ainappnotification", conn);
                 await EraseTableAsync("anotifyevent", conn);
                 await EraseTableAsync("anotifydeliverylog", conn);
                 await EraseTableAsync("anotifysubscription", conn);