server/DataList/SubscriptionServerDataList.cs

@@ -14,7 +14,7 @@ namespace Sockeye.DataList
 
             var RoleSet = BizRoles.GetRoleSet(DefaultListAType);
             AllowedRoles = RoleSet.ReadFullRecord | RoleSet.Change;
-            DefaultColumns = new List<string>() { "SubServerSubExpire", "SubServerName", "Customer", "SubServerTrial", "SubServerDatacenter", "ServerState" };
+            DefaultColumns = new List<string>() { "SubServerSubExpire", "SubServerName", "Customer", "SubServerTrial", "SubServerLastHealthStatus", "SubServerLastHealthCheck", "SubServerDatacenter", "ServerState" };
             DefaultSortBy = new Dictionary<string, string>() { { "SubServerSubExpire", "+" } };
             FieldDefinitions = new List<DataListFieldDefinition>();
 
@@ -95,6 +95,22 @@ namespace Sockeye.DataList
                 SqlValueColumnName = "asubscriptionserver.subscriptionexpire"
             });
 
+            FieldDefinitions.Add(new DataListFieldDefinition
+            {
+                TKey = "SubServerLastHealthStatus",
+                FieldKey = "SubServerLastHealthStatus",
+                UiFieldDataType = (int)UiFieldDataType.Text,
+                SqlValueColumnName = "asubscriptionserver.lasthealthstatus"
+            });
+
+            FieldDefinitions.Add(new DataListFieldDefinition
+            {
+                TKey = "SubServerLastHealthCheck",
+                FieldKey = "SubServerLastHealthCheck",
+                UiFieldDataType = (int)UiFieldDataType.DateTime,
+                SqlValueColumnName = "asubscriptionserver.lasthealthcheck"
+            });
+
             FieldDefinitions.Add(new DataListFieldDefinition
             {
                 TKey = "SubServerTrial",

server/biz/JobsBiz.cs

@@ -260,9 +260,13 @@ namespace Sockeye.Biz
                 await CoreNotificationSweeper.DoWorkAsync();
                 if (!KeepOnWorking()) return;
 
-               
+                //SOCKBOT - SUBSCRIPTION SERVER HEALTH CHECKS
+                
 
-                //JOB SWEEPER / AND USER COUNT CHECK
+
+
+               
+                //JOB SWEEPER 
                 await CoreJobSweeper.DoWorkAsync();
                 if (!KeepOnWorking()) return;
 

server/generator/CoreJobSweeper.cs

@@ -8,7 +8,7 @@ using Sockeye.Models;
 namespace Sockeye.Biz
 {
 
-    //#################### NOTE: This also does some license checking tasks, put in here instead of the corejoblicense job deliberately #############
+   
     /// <summary>
     /// JobSweeper - called by Generator to clean out old jobs that are completed and their logs
     /// 

server/generator/SockBotSubscriptionServerHealthChecks.cs

@@ -0,0 +1,124 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using Sockeye.Models;
+
+namespace Sockeye.Biz
+{
+
+   
+    /// <summary>
+    /// Check the health of subscribers servers, basically a ping check
+    /// trigger notification if any fail the test excessively (some slack for intermittent comm. issues)
+    /// </summary>
+    internal static class SockBotSubscriptionServerHealthChecks
+    {
+        private static ILogger log = Sockeye.Util.ApplicationLogging.CreateLogger("SockBotSubscriptionServerHealthChecks");
+        private static DateTime lastSweep = DateTime.MinValue;
+        private static TimeSpan HEALTHCHECK_EVERY_INTERVAL = new TimeSpan(0, 15, 10);//every 15 minutes roughly
+        ////////////////////////////////////////////////////////////////////////////////////////////////
+        // DoSweep
+        //
+        public static async Task DoWorkAsync()
+        {
+            //This will get triggered roughly every minute, but we don't want to check that frequently
+            if (DateTime.UtcNow - lastSweep < HEALTHCHECK_EVERY_INTERVAL)
+                return;
+
+            log.LogDebug("Health check starting");
+            using (AyContext ct = Sockeye.Util.ServiceProviderProvider.DBContext)
+            {
+              
+  var servers = await ct.SubscriptionServer
+                   .AsNoTracking()
+                   .Where(z => z.ServerState==  < dtDeleteCutoff && z.JobStatus == jobStatus)
+                   .OrderBy(z => z.Created)
+                   .ToListAsync();
+
+            }
+            lastSweep = DateTime.UtcNow;
+        }
+
+
+        private static async Task sweepAsync(AyContext ct, DateTime dtDeleteCutoff, JobStatus jobStatus)
+        {
+            //Get the deleteable succeeded jobs list
+          
+
+            log.LogDebug($"SweepAsync processing: cutoff={dtDeleteCutoff.ToString()}, for {jobs.Count.ToString()} jobs of status {jobStatus.ToString()}");
+
+            foreach (OpsJob j in jobs)
+            {
+                try
+                {
+
+                    await JobsBiz.RemoveJobAndLogsAsync(j.GId);
+                }
+                catch (Exception ex)
+                {
+                    log.LogError(ex, "sweepAsync exception calling JobsBiz.RemoveJobAndLogsAsync");
+                    //for now just throw it but this needs to be removed when logging added and better handling 
+                    throw;
+                }
+            }
+        }
+
+
+        /// <summary>
+        /// Kill jobs that have been stuck in "running" state for too long
+        /// </summary>        
+        private static async Task killStuckJobsAsync(AyContext ct, DateTime dtRunningDeadline)
+        {
+            //Get the deleteable succeeded jobs list
+            var jobs = await ct.OpsJob
+                   .AsNoTracking()
+                   .Where(z => z.Created < dtRunningDeadline && z.JobStatus == JobStatus.Running)
+                   .OrderBy(z => z.Created)
+                   .ToListAsync();
+
+            log.LogDebug($"killStuckJobsAsync processing: cutoff={dtRunningDeadline.ToString()}, for {jobs.Count.ToString()} jobs of status {JobStatus.Running.ToString()}");
+
+            foreach (OpsJob j in jobs)
+            {
+                //OPSMETRIC
+                await JobsBiz.LogJobAsync(j.GId, "LT:JobFailed LT:TimedOut");
+                log.LogError($"Job found job stuck in running status and set to failed: deadline={dtRunningDeadline.ToString()}, jobId={j.GId.ToString()}, jobname={j.Name}, jobtype={j.JobType.ToString()}, jobAType={j.SockType.ToString()}, jobObjectId={j.ObjectId.ToString()}");
+                await JobsBiz.UpdateJobStatusAsync(j.GId, JobStatus.Failed);
+            }
+        }
+
+
+        private static async Task SweepInternalJobsLogsAsync(AyContext ct, DateTime dtDeleteCutoff)
+        {
+            //Get the deleteable list (this is for reporting, could easily just do it in one go)
+            var logs = await ct.OpsJobLog
+                   .AsNoTracking()
+                   .Where(z => z.Created < dtDeleteCutoff)
+                   .OrderBy(z => z.Created)
+                   .ToListAsync();
+
+            log.LogDebug($"SweepInternalJobsLogsAsync processing: cutoff={dtDeleteCutoff.ToString()}, for {logs.Count.ToString()} log entries");
+
+            foreach (OpsJobLog l in logs)
+            {
+                try
+                {
+                    await ct.Database.ExecuteSqlInterpolatedAsync($"delete from aopsjoblog where gid = {l.GId}");
+                }
+                catch (Exception ex)
+                {
+                    log.LogError(ex, "SweepInternalJobsLogsAsync exception removed old log entries");
+                    throw;
+                }
+            }
+        }
+
+        /////////////////////////////////////////////////////////////////////
+
+    }//eoc
+
+
+}//eons
+

server/models/SubscriptionServer.cs

@@ -27,6 +27,8 @@ namespace Sockeye.Models
         public DateTime? LastUpdated { get; set; }
         public DateTime SubscriptionExpire { get; set; }
         public decimal Cost { get; set; }
+        public string LastHealthStatus { get; set; }
+        public DateTime? LastHealthCheck { get; set; }
         public bool Trial { get; set; }
         public string TrialContact { get; set; }
         public string TrialEmail { get; set; }

server/util/AySchema.cs

@@ -888,6 +888,7 @@ $BODY$ LANGUAGE PLPGSQL STABLE");
 
                 await ExecQueryAsync("CREATE TABLE asubscriptionserver (id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, serverstate INTEGER NOT NULL DEFAULT 0, created TIMESTAMPTZ NOT NULL, "
                                   + "name TEXT NOT NULL, ipaddress TEXT, notes TEXT, datacenter TEXT NOT NULL, timezone TEXT NOT NULL, dbid TEXT, lastupdated TIMESTAMPTZ, subscriptionexpire TIMESTAMPTZ NOT NULL, "
+                                  + "lasthealthstatus TEXT, lasthealthcheck TIMESTAMPTZ, "
                                   + "trial BOOL NOT NULL DEFAULT true, trialcontact TEXT, trialemail TEXT, trialcompany TEXT, operatingsystem TEXT, customersubdomain TEXT, cost DECIMAL(38,18) NOT NULL default 7, "
                                   + "wiki TEXT, tags VARCHAR(255) ARRAY, customerid BIGINT REFERENCES acustomer(id) )");
 
@@ -1047,7 +1048,9 @@ $BODY$ LANGUAGE PLPGSQL STABLE");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerDatacenter', 'Data center' FROM atranslation t  where t.baselanguage = 'en'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTimeZone', 'Time zone' FROM atranslation t  where t.baselanguage = 'en'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastUpdated', 'Last updated' FROM atranslation t  where t.baselanguage = 'en'");
-                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerSubExpire', 'Subscription expires' FROM atranslation t  where t.baselanguage = 'en'");
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerSubExpire', 'Subscription expires' FROM atranslation t  where t.baselanguage = 'en'");                
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastHealthStatus', 'Health status' FROM atranslation t  where t.baselanguage = 'en'");
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastHealthCheck', 'Last health check' FROM atranslation t  where t.baselanguage = 'en'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrialContact', 'Trial contact' FROM atranslation t  where t.baselanguage = 'en'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrialEmail', 'TrialEmail' FROM atranslation t  where t.baselanguage = 'en'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrial', 'Trial' FROM atranslation t  where t.baselanguage = 'en'");
@@ -1072,7 +1075,9 @@ $BODY$ LANGUAGE PLPGSQL STABLE");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerDatacenter', 'Data center' FROM atranslation t  where t.baselanguage = 'es'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTimeZone', 'Time zone' FROM atranslation t  where t.baselanguage = 'es'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastUpdated', 'Last updated' FROM atranslation t  where t.baselanguage = 'es'");
-                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerSubExpire', 'Subscription expires' FROM atranslation t  where t.baselanguage = 'es'");
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerSubExpire', 'Subscription expires' FROM atranslation t  where t.baselanguage = 'es'");              
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastHealthStatus', 'Health status' FROM atranslation t  where t.baselanguage = 'es'");
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastHealthCheck', 'Last health check' FROM atranslation t  where t.baselanguage = 'es'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrialContact', 'Trial contact' FROM atranslation t  where t.baselanguage = 'es'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrialEmail', 'TrialEmail' FROM atranslation t  where t.baselanguage = 'es'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrial', 'Trial' FROM atranslation t  where t.baselanguage = 'es'");
@@ -1097,7 +1102,9 @@ $BODY$ LANGUAGE PLPGSQL STABLE");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerDatacenter', 'Data center' FROM atranslation t  where t.baselanguage = 'fr'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTimeZone', 'Time zone' FROM atranslation t  where t.baselanguage = 'fr'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastUpdated', 'Last updated' FROM atranslation t  where t.baselanguage = 'fr'");
-                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerSubExpire', 'Subscription expires' FROM atranslation t  where t.baselanguage = 'fr'");
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerSubExpire', 'Subscription expires' FROM atranslation t  where t.baselanguage = 'fr'");              
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastHealthStatus', 'Health status' FROM atranslation t  where t.baselanguage = 'fr'");
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastHealthCheck', 'Last health check' FROM atranslation t  where t.baselanguage = 'fr'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrialContact', 'Trial contact' FROM atranslation t  where t.baselanguage = 'fr'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrialEmail', 'TrialEmail' FROM atranslation t  where t.baselanguage = 'fr'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrial', 'Trial' FROM atranslation t  where t.baselanguage = 'fr'");
@@ -1122,7 +1129,9 @@ $BODY$ LANGUAGE PLPGSQL STABLE");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerDatacenter', 'Data center' FROM atranslation t  where t.baselanguage = 'de'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTimeZone', 'Time zone' FROM atranslation t  where t.baselanguage = 'de'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastUpdated', 'Last updated' FROM atranslation t  where t.baselanguage = 'de'");
-                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerSubExpire', 'Subscription expires' FROM atranslation t  where t.baselanguage = 'de'");
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerSubExpire', 'Subscription expires' FROM atranslation t  where t.baselanguage = 'de'");              
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastHealthStatus', 'Health status' FROM atranslation t  where t.baselanguage = 'de'");
+                await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerLastHealthCheck', 'Last health check' FROM atranslation t  where t.baselanguage = 'de'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrialContact', 'Trial contact' FROM atranslation t  where t.baselanguage = 'de'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrialEmail', 'TrialEmail' FROM atranslation t  where t.baselanguage = 'de'");
                 await ExecQueryAsync("INSERT INTO atranslationitem(translationid,key,display) SELECT t.id, 'SubServerTrial', 'Trial' FROM atranslation t  where t.baselanguage = 'de'");

todo.txt

@@ -1,24 +1,13 @@
 TODO:
 
-DOCS
-Once it's up and running update docs to mention that perpetual trial period is two weeks subscription is one week
 
-
-- Purchase event must trigger notification event subscribable (might already due to created etc)
-
-
-    make sure to add 
-            if (ServerBootConfig.MIGRATING) return;
-    check or all hell will break loose
-    Also wrap the notification in a DEBUG block so that it will send to gzmailadmin only instead of customer actual email 
-    this way can really test out anything without fear
-
-
-
-- trial server request route that contact form can trigger
 - JOB: notify user active license
-- JOB: vendor notification to purchase, create customer if necessary from purchase notification or add to existing
+- JOB: Process purchases that are from vendor notification
-    - sub or part of above JOB: purchase to license
+    - create customer if necessary from purchase notification or add to existing
+    - if v7 needs to account for there being a delay sometimes in order completeness
+        maybe just keep adding to the order and refreshing 
+- JOB: make a license from purchases but not active ready to send / in waiting only for approval
+    
 - JOB: Ping / check health route of subscription server
     flag last health check
     trigger event notification if fails
@@ -61,114 +50,26 @@ Once it's up and running update docs to mention that perpetual trial period is t
 
 
 - Add ui to front the route under /api/v8/license/v7-reset-manager-creds
-- Need ui that shows money required for paying future for yearly subs so at a glance know what money need in bank each month
-
-
-////////////////////////////////////////////////////////////////////////////////////////////
-//OLD 
-
-LICENSE NOTES
-
-
-
-------
-
-Sockeye should generate a license on a sale receipt, maybe after a slight delay. 
-Generated license should be in the list of licenses but have a unfulfilled status of some kind so it's ready for me to visually 
-see purchases applicable and confirm approve and send it with one click but it's not doing it itself.
-
-If more purchases come in that apply to same customer for v7 then it should regenerate a new license or I guess edit the one it made before and just add to it.
-Idea being that all I have to do is open they list of licenses and just release any that are good to go.
-UI should show the purchases that go into a license so can confirm visually without having to click anywhere else.
-
-This way is all but automatic in preparation for full automation later.
-Maybe even a semi automatic dead man switch that will just send it after 24 guess if I don't once it's confirmed to work properly.
-
-So need properties added to license object for fulfilled status and a property indicating sent to customer and a menu options to send manually maybe too.
-
-I guess will need a template for email messages as well.
-Tied into notification system.
-Don't forget some licenses are not done through SHAREitb do must support manual generation from Paypal etc
-
-ALSO 
-
-Needs manual license generation for v7 still
-
-
-** CONTACT FORM 
-Server request for trial subscription server should maybe go through sockeye instead as a form people can request from or the contact app should forward to sockeye
-so it can create a new subscription server record that is pending status for me to just approve and ultimately auto-generate a server using D.O. API or whatever
-
-
-
-=========
-
-PURCHASE drives new licensing ui
-To automate as much as possible need following:
-- Purchase Customer id nullable as may not be able to match to an existing customer
-- processed date important as shows what is not processed yet if empty
-- product category for purchase automatically attributed, i.e. v7, raven perpet, raven sub, misc
-
-
-License edit / entry form
-- Pick license key type different fields appear
-    NEED TO ADD KEYTYPE ENUM TO LICENSE OBJECT
-        this will future proof and make UI and handling easier
-        Keytypes: AyaNova 7, RAVEN Perpetual, RAVEN Subscription
-- generates key at server on save depending on what is new or changed I guess or needs to be edited after the fact?
-- Generate v7 license from entered ad-hoc data, i.e. it should be from the license edit form and take the entries and make a key on save
-- Generate v7 license for customer from all active purchases
-    NEED TO ADD PRODUCT CATEGORY FOR AUTO LICENSE GENERATION?
-        V7, RAVEN, MISC (not license keyed)
-        this way, can automatically create key for v7 from all purchases that have not yet expired for a customer
-- Generate v8 license from entered data, ''
-- Opening license existing in edit form should re-populate the controls even for v7 so that they can be changed, saved, edited etc
-- Once it's fetched it's read-only but can duplicate!!!!!
 
 - direct open a case by case number like workorder
 (id's differ so it isn't easy to just open a case in the url)
 
 
-NOTE: v7 licensing will be hard to automate fully as it's a fucked up system
+LATER
-RAVEN is much easier to automate with only one key per db
-So I'm thinking make v7 renewals and purchases as easy and semiautomatic as possible, perhaps it fills in and suggest the whole thing but requires me
-to press a button to actually do it, whereas raven can ultimately just process automatically, particularly subscription monthly important!!
 
-AUTOMATION ROUTES REQUIRED
+- ** DOCS Once it's up and running update docs to mention that perpetual trial period is two weeks subscription is one week
-Both v7 and v8 point to "rockfish.ayanova.com" so maybe nginx can redirect?
+- trial server request route that contact form can trigger
-forgot about that, was hoping for ability to parallel and keep rockfish going... hmmm...
+- Subscription server trial request form through Sockeye instead?
-
+    or perhaps it sends to sockeye route?
-AyaNova 7 license fetch URL
+    this allows for automation and notification handling and auto generation
-"https://rockfish.ayanova.com/fetch/" + sFetchCode + "/" + sEmail;
+- Contact form via sockeye instead?
-rockfish FetchController.cs
+    or perhaps it sends to sockey contact route?
-
+    this allows for automation and notification handling and auto reply once qualified if a customer or not
-RAVEN trial request URL
+- Need ui that shows money required for paying future for yearly subs so at a glance know what money need in bank each month
-POST to 
-$"{LICENSE_SERVER_URL_ROCKFISH}rvr";
-
-RAVEN license fetch url
-POST to
- $"{LicenseServer}/rvf"
-
- License server url is one of:
 
 
-LICENSE_SERVER_URL_ROCKFISH = "http://localhost:3001/";//dev testing
+////////////////////////////////////////////////////////////////////////////////////////////
-LICENSE_SERVER_URL_ROCKFISH = "https://rockfish.ayanova.com/";//1st default production primary
+//OLD 
-LICENSE_SERVER_URL_IO = "https://io.ayanova.com/";//2nd production
-LICENSE_SERVER_URL_EUROPA = "https://europa.ayanova.com/";//3rd production
-LICENSE_SERVER_URL_CALLISTO = "https://callisto.ayanova.com/";//4th production
-
-
-DTR
-
-Also, sidebar, check if can pay d.o. in advance, or do I want that??
-
-
-new roles and bizroles set appropo
-    Developer - rw cases
-    Support - read cases, read sales and customer data?
-    Sales - licenses