devdocs/todo.txt

@@ -9,6 +9,7 @@ Problem:
 
 potential fixes: TODO HERE:
 	Detect postgres is unreachable and stop digging a hole, wait to resolve it before continuing on again
+	
 	Don't log same message repeatedly
 		actually this is not necessary if just stop running jobs when can't reach sql server
 	FIXED don't dual log to system journal logs and ayanova log

server/AyaNova/Program.cs

@@ -30,6 +30,9 @@ namespace AyaNova
             //Boot lock for generator
             ServerGlobalOpsSettingsCache.BOOTING = true;
 
+            //preset, we don't know yet
+            ServerGlobalOpsSettingsCache.DBAVAILABLE = false;
+
             //Get config
             var config = new ConfigurationBuilder().AddJsonFile("config.json", true).AddEnvironmentVariables().AddCommandLine(args).Build();
 

server/AyaNova/Startup.cs

@@ -142,6 +142,8 @@ namespace AyaNova
             }
 
 
+            //We have db available
+            ServerGlobalOpsSettingsCache.DBAVAILABLE = true;
 
             _newLog.LogInformation("Connected to database server - {0}", DbUtil.DisplayableConnectionString);
 

server/AyaNova/util/DbUtil.cs

@@ -202,10 +202,34 @@ namespace AyaNova.Util
                 return false;
             }
             return true;
-
         }
 
-
+        ///////////////////////////////////////////////
+        // Set global flag if db server is connectable
+        //
+        internal static void CheckDatabaseServerAvailable()
+        {
+            //Called by generator when db is down to check if it can connect
+            //should not run too quickly or too often so a delay is built in here
+            if (DateTime.UtcNow - LAST_CHECK_IF_DB_SERVER_AVAILABLE < CHECK_IF_DB_SERVER_AVAILABLE_DELAY)//no more often than every 10 seconds
+                return;
+            try
+            {
+                using (var conn = new Npgsql.NpgsqlConnection(AdminConnectionString))
+                {
+                    conn.Open();
+                    conn.Close();
+                }
+            }
+            catch
+            {
+                return;
+            }
+            //We have db available
+            ServerGlobalOpsSettingsCache.DBAVAILABLE = true;
+        }
+        private static DateTime LAST_CHECK_IF_DB_SERVER_AVAILABLE = DateTime.MinValue;
+        private static TimeSpan CHECK_IF_DB_SERVER_AVAILABLE_DELAY = new TimeSpan(0, 0, 10);
 
 
         ///////////////////////////////////////////

server/AyaNova/util/ServerGlobalOpsSettingsCache.cs

@@ -15,6 +15,10 @@ namespace AyaNova.Util
         //is used to control generator from starting
         internal static bool BOOTING { get; set; }
 
+        //False if db server is detected to be down
+        //is used to control generator from processing
+        internal static bool DBAVAILABLE { get; set; }
+
         internal static GlobalOpsBackupSettings Backup { get; set; }
         internal static GlobalOpsNotificationSettings Notify { get; set; }
         internal static DateTime NextBackup { get; set; }