server/DataList/SubscriptionServerDataList.cs

@@ -14,7 +14,7 @@ namespace Sockeye.DataList
 
             var RoleSet = BizRoles.GetRoleSet(DefaultListAType);
             AllowedRoles = RoleSet.ReadFullRecord | RoleSet.Change;
-            DefaultColumns = new List<string>() { "SubServerSubExpire", "SubServerName", "Customer", "SubServerTrial", "SubServerDatacenter", "ServerState" };
+            DefaultColumns = new List<string>() { "SubServerSubExpire", "SubServerName", "Customer", "SubServerTrial", "SubServerLastHealthStatus", "SubServerLastHealthCheck", "SubServerDatacenter", "ServerState" };
             DefaultSortBy = new Dictionary<string, string>() { { "SubServerSubExpire", "+" } };
             FieldDefinitions = new List<DataListFieldDefinition>();
 
@@ -95,6 +95,22 @@ namespace Sockeye.DataList
                 SqlValueColumnName = "asubscriptionserver.subscriptionexpire"
             });
 
+            FieldDefinitions.Add(new DataListFieldDefinition
+            {
+                TKey = "SubServerLastHealthStatus",
+                FieldKey = "SubServerLastHealthStatus",
+                UiFieldDataType = (int)UiFieldDataType.Text,
+                SqlValueColumnName = "asubscriptionserver.lasthealthstatus"
+            });
+
+            FieldDefinitions.Add(new DataListFieldDefinition
+            {
+                TKey = "SubServerLastHealthCheck",
+                FieldKey = "SubServerLastHealthCheck",
+                UiFieldDataType = (int)UiFieldDataType.DateTime,
+                SqlValueColumnName = "asubscriptionserver.lasthealthcheck"
+            });
+
             FieldDefinitions.Add(new DataListFieldDefinition
             {
                 TKey = "SubServerTrial",

server/biz/JobsBiz.cs

@@ -260,9 +260,13 @@ namespace Sockeye.Biz
                 await CoreNotificationSweeper.DoWorkAsync();
                 if (!KeepOnWorking()) return;
 
-               
+                //SOCKBOT - SUBSCRIPTION SERVER HEALTH CHECKS
+                
 
-                //JOB SWEEPER / AND USER COUNT CHECK
+
+
+               
+                //JOB SWEEPER 
                 await CoreJobSweeper.DoWorkAsync();
                 if (!KeepOnWorking()) return;
 

server/generator/CoreJobSweeper.cs

@@ -8,7 +8,7 @@ using Sockeye.Models;
 namespace Sockeye.Biz
 {
 
-    //#################### NOTE: This also does some license checking tasks, put in here instead of the corejoblicense job deliberately #############
+   
     /// <summary>
     /// JobSweeper - called by Generator to clean out old jobs that are completed and their logs
     /// 

server/generator/SockBotSubscriptionServerHealthChecks.cs

@@ -0,0 +1,124 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using Sockeye.Models;
+
+namespace Sockeye.Biz
+{
+
+   
+    /// <summary>
+    /// Check the health of subscribers servers, basically a ping check
+    /// trigger notification if any fail the test excessively (some slack for intermittent comm. issues)
+    /// </summary>
+    internal static class SockBotSubscriptionServerHealthChecks
+    {
+        private static ILogger log = Sockeye.Util.ApplicationLogging.CreateLogger("SockBotSubscriptionServerHealthChecks");
+        private static DateTime lastSweep = DateTime.MinValue;
+        private static TimeSpan HEALTHCHECK_EVERY_INTERVAL = new TimeSpan(0, 15, 10);//every 15 minutes roughly
+        ////////////////////////////////////////////////////////////////////////////////////////////////
+        // DoSweep
+        //
+        public static async Task DoWorkAsync()
+        {
+            //This will get triggered roughly every minute, but we don't want to check that frequently
+            if (DateTime.UtcNow - lastSweep < HEALTHCHECK_EVERY_INTERVAL)
+                return;
+
+            log.LogDebug("Health check starting");
+            using (AyContext ct = Sockeye.Util.ServiceProviderProvider.DBContext)
+            {
+              
+  var servers = await ct.SubscriptionServer
+                   .AsNoTracking()
+                   .Where(z => z.ServerState==  < dtDeleteCutoff && z.JobStatus == jobStatus)
+                   .OrderBy(z => z.Created)
+                   .ToListAsync();
+
+            }
+            lastSweep = DateTime.UtcNow;
+        }
+
+
+        private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus)
+        {
+            //Get the deleteable succeeded jobs list
+          
+
+            log.LogDebug($"SweepAsync processing: cutoff={dtDeleteCutoff.ToString()}, for {jobs.Count.ToString()} jobs of status {jobStatus.ToString()}");
+
+            foreach (OpsJob j in jobs)
+            {
+                try
+                {
+
+                    await JobsBiz.RemoveJobAndLogsAsync(j.GId);
+                }
+                catch (Exception ex)
+                {
+                    log.LogError(ex, "sweepAsync exception calling JobsBiz.RemoveJobAndLogsAsync");
+                    //for now just throw it but this needs to be removed when logging added and better handling 
+                    throw;
+                }
+            }
+        }
+
+
+        /// <summary>
+        /// Kill jobs that have been stuck in "running" state for too long
+        /// </summary>        
+        private static async Task killStuckJobsAsync(AyContext ct, DateTime dtRunningDeadline)
+        {
+            //Get the deleteable succeeded jobs list
+            var jobs = await ct.OpsJob
+                   .AsNoTracking()
+                   .Where(z => z.Created < dtRunningDeadline && z.JobStatus == JobStatus.Running)
+                   .OrderBy(z => z.Created)
+                   .ToListAsync();
+
+            log.LogDebug($"killStuckJobsAsync processing: cutoff={dtRunningDeadline.ToString()}, for {jobs.Count.ToString()} jobs of status {JobStatus.Running.ToString()}");
+
+            foreach (OpsJob j in jobs)
+            {
+                //OPSMETRIC
+                await JobsBiz.LogJobAsync(j.GId, "LT:JobFailed LT:TimedOut");
+                log.LogError($"Job found job stuck in running status and set to failed: deadline={dtRunningDeadline.ToString()}, jobId={j.GId.ToString()}, jobname={j.Name}, jobtype={j.JobType.ToString()}, jobAType={j.SockType.ToString()}, jobObjectId={j.ObjectId.ToString()}");
+                await JobsBiz.UpdateJobStatusAsync(j.GId, JobStatus.Failed);
+            }
+        }
+
+
+        private static async Task SweepInternalJobsLogsAsync(AyContext ct, DateTime dtDeleteCutoff)
+        {
+            //Get the deleteable list (this is for reporting, could easily just do it in one go)
+            var logs = await ct.OpsJobLog
+                   .AsNoTracking()
+                   .Where(z => z.Created < dtDeleteCutoff)
+                   .OrderBy(z => z.Created)
+                   .ToListAsync();
+
+            log.LogDebug($"SweepInternalJobsLogsAsync processing: cutoff={dtDeleteCutoff.ToString()}, for {logs.Count.ToString()} log entries");
+
+            foreach (OpsJobLog l in logs)
+            {
+                try
+                {
+                    await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjoblog where gid = {l.GId}");
+                }
+                catch (Exception ex)
+                {
+                    log.LogError(ex, "SweepInternalJobsLogsAsync exception removed old log entries");
+                    throw;
+                }
+            }
+        }
+
+        /////////////////////////////////////////////////////////////////////
+
+    }//eoc
+
+
+}//eons
+

server/models/SubscriptionServer.cs

@@ -27,6 +27,8 @@ namespace Sockeye.Models
         public DateTime? LastUpdated { get; set; }
         public DateTime SubscriptionExpire { get; set; }
         public decimal Cost { get; set; }
+        public string LastHealthStatus { get; set; }
+        public DateTime? LastHealthCheck { get; set; }
         public bool Trial { get; set; }
         public string TrialContact { get; set; }
         public string TrialEmail { get; set; }

server/util/AySchema.cs

@@ -888,6 +888,7 @@ $BODY$ LANGUAGE PLPGSQL STABLE");
 
                 await ExecQueryAsync("CREATE TABLE asubscriptionserver (id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, serverstate INTEGER NOT NULL DEFAULT 0, created TIMESTAMPTZ NOT NULL, "
                                   + "name TEXT NOT NULL, ipaddress TEXT, notes TEXT, datacenter TEXT NOT NULL, timezone TEXT NOT NULL, dbid TEXT, lastupdated TIMESTAMPTZ, subscriptionexpire TIMESTAMPTZ NOT NULL, "
+                                  + "lasthealthstatus TEXT, lasthealthcheck TIMESTAMPTZ, "
                                   + "trial BOOL NOT NULL DEFAULT true, trialcontact TEXT, trialemail TEXT, trialcompany TEXT, operatingsystem TEXT, customersubdomain TEXT, cost DECIMAL(38,18) NOT NULL default 7, "
                                   + "wiki TEXT, tags VARCHAR(255) ARRAY, customerid BIGINT REFERENCES acustomer(id) )");
 
@@ -1047,7 +1048,9 @@ $BODY$ LANGUAGE PLPGSQL STABLE");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerDatacenter', 'Data center' FROM atranslation t  where t.baselanguage = 'en'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTimeZone', 'Time zone' FROM atranslation t  where t.baselanguage = 'en'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastUpdated', 'Last updated' FROM atranslation t  where t.baselanguage = 'en'");
-                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerSubExpire', 'Subscription expires' FROM atranslation t  where t.baselanguage = 'en'");
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerSubExpire', 'Subscription expires' FROM atranslation t  where t.baselanguage = 'en'");                
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastHealthStatus', 'Health status' FROM atranslation t  where t.baselanguage = 'en'");
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastHealthCheck', 'Last health check' FROM atranslation t  where t.baselanguage = 'en'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrialContact', 'Trial contact' FROM atranslation t  where t.baselanguage = 'en'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrialEmail', 'TrialEmail' FROM atranslation t  where t.baselanguage = 'en'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrial', 'Trial' FROM atranslation t  where t.baselanguage = 'en'");
@@ -1072,7 +1075,9 @@ $BODY$ LANGUAGE PLPGSQL STABLE");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerDatacenter', 'Data center' FROM atranslation t  where t.baselanguage = 'es'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTimeZone', 'Time zone' FROM atranslation t  where t.baselanguage = 'es'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastUpdated', 'Last updated' FROM atranslation t  where t.baselanguage = 'es'");
-                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerSubExpire', 'Subscription expires' FROM atranslation t  where t.baselanguage = 'es'");
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerSubExpire', 'Subscription expires' FROM atranslation t  where t.baselanguage = 'es'");              
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastHealthStatus', 'Health status' FROM atranslation t  where t.baselanguage = 'es'");
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastHealthCheck', 'Last health check' FROM atranslation t  where t.baselanguage = 'es'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrialContact', 'Trial contact' FROM atranslation t  where t.baselanguage = 'es'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrialEmail', 'TrialEmail' FROM atranslation t  where t.baselanguage = 'es'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrial', 'Trial' FROM atranslation t  where t.baselanguage = 'es'");
@@ -1097,7 +1102,9 @@ $BODY$ LANGUAGE PLPGSQL STABLE");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerDatacenter', 'Data center' FROM atranslation t  where t.baselanguage = 'fr'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTimeZone', 'Time zone' FROM atranslation t  where t.baselanguage = 'fr'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastUpdated', 'Last updated' FROM atranslation t  where t.baselanguage = 'fr'");
-                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerSubExpire', 'Subscription expires' FROM atranslation t  where t.baselanguage = 'fr'");
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerSubExpire', 'Subscription expires' FROM atranslation t  where t.baselanguage = 'fr'");              
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastHealthStatus', 'Health status' FROM atranslation t  where t.baselanguage = 'fr'");
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastHealthCheck', 'Last health check' FROM atranslation t  where t.baselanguage = 'fr'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrialContact', 'Trial contact' FROM atranslation t  where t.baselanguage = 'fr'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrialEmail', 'TrialEmail' FROM atranslation t  where t.baselanguage = 'fr'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrial', 'Trial' FROM atranslation t  where t.baselanguage = 'fr'");
@@ -1122,7 +1129,9 @@ $BODY$ LANGUAGE PLPGSQL STABLE");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerDatacenter', 'Data center' FROM atranslation t  where t.baselanguage = 'de'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTimeZone', 'Time zone' FROM atranslation t  where t.baselanguage = 'de'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastUpdated', 'Last updated' FROM atranslation t  where t.baselanguage = 'de'");
-                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerSubExpire', 'Subscription expires' FROM atranslation t  where t.baselanguage = 'de'");
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerSubExpire', 'Subscription expires' FROM atranslation t  where t.baselanguage = 'de'");              
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastHealthStatus', 'Health status' FROM atranslation t  where t.baselanguage = 'de'");
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastHealthCheck', 'Last health check' FROM atranslation t  where t.baselanguage = 'de'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrialContact', 'Trial contact' FROM atranslation t  where t.baselanguage = 'de'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrialEmail', 'TrialEmail' FROM atranslation t  where t.baselanguage = 'de'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrial', 'Trial' FROM atranslation t  where t.baselanguage = 'de'");