server/AyaNova/biz/JobsBiz.cs

@@ -186,6 +186,14 @@ namespace AyaNova.Biz
                 return;
             }
 
+            //Do not process if there is no db, everything relies on it below here
+            if (!ServerGlobalOpsSettingsCache.DBAVAILABLE)
+            {
+                //This will set dbavailable flag if it becomes available
+                DbUtil.CheckDatabaseServerAvailable();
+                return;
+            }
+
             ActivelyProcessing = true;
             log.LogTrace("Processing internal jobs");
             try
@@ -307,6 +315,7 @@ namespace AyaNova.Biz
             {
                 var msg = "Server::ProcessJobsAsync unexpected error during processing";
                 log.LogError(ex, msg);
+                DbUtil.HandleIfDatabaseUnavailableTypeException(ex);
                 await NotifyEventHelper.AddOpsProblemEvent(msg, ex);
             }
             finally

server/AyaNova/biz/NotifyEventHelper.cs

@@ -395,6 +395,7 @@ namespace AyaNova.Biz
             catch (Exception ex)
             {
                 log.LogError(ex, $"Error adding general notify event [type:{eventType}, userId:{userId}, message:{message}]");
+                DbUtil.HandleIfDatabaseUnavailableTypeException(ex);
             }
 
         }//eom

server/AyaNova/biz/PMBiz.cs

@@ -5211,6 +5211,7 @@ namespace AyaNova.Biz
                 catch (Exception ex)
                 {
                     log.LogError(ex, $"error generating Work order from PM {p.Serial}");
+                    DbUtil.HandleIfDatabaseUnavailableTypeException(ex);
                     await NotifyEventHelper.AddGeneralNotifyEvent(AyaType.PM, p.Id, NotifyEventType.PMGenerationFailed, $"Error generating Work order from PM {p.Serial}", "Preventive Maintenance", ex);
                     if (!KeepOnWorking(log)) return;
                     continue;

server/AyaNova/generator/CoreJobCustomerNotify.cs

@@ -100,6 +100,7 @@ namespace AyaNova.Biz
             catch (Exception ex)
             {
                 log.LogError(ex, $"Error processing customer notification event");
+                DbUtil.HandleIfDatabaseUnavailableTypeException(ex);
             }
             finally
             {

server/AyaNova/generator/CoreJobNotify.cs

@@ -141,6 +141,7 @@ namespace AyaNova.Biz
             catch (Exception ex)
             {
                 log.LogError(ex, $"Error processing notification event");
+                DbUtil.HandleIfDatabaseUnavailableTypeException(ex);
             }
             finally
             {

server/AyaNova/generator/CoreJobPMGenerate.cs

@@ -56,7 +56,9 @@ namespace AyaNova.Biz
             catch (Exception ex)
             {
                 log.LogError(ex, $"Error processing PMGenerate ");
-                 await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.PMGenerationFailed, "PM Generate failure","Preventive Maintenance", ex);
+                DbUtil.HandleIfDatabaseUnavailableTypeException(ex);
+                await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.PMGenerationFailed, "PM Generate failure", "Preventive Maintenance", ex);
+
             }
             finally
             {

server/AyaNova/generator/CoreJobPMInventoryCheck.cs

@@ -58,6 +58,7 @@ namespace AyaNova.Biz
             catch (Exception ex)
             {
                 log.LogError(ex, $"Error processing CoreJobPMInventoryCheck ");
+                DbUtil.HandleIfDatabaseUnavailableTypeException(ex);
                 await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.PMGenerationFailed, "PM Inventory check failure", "Preventive Maintenance", ex);
             }
             finally

server/AyaNova/util/DbUtil.cs

@@ -232,6 +232,28 @@ namespace AyaNova.Util
         private static TimeSpan CHECK_IF_DB_SERVER_AVAILABLE_DELAY = new TimeSpan(0, 0, 10);
 
 
+        ///////////////////////////////////////////////////////////
+        // Check if exception means db server is unavailable
+        // if so, flag global flag indicating it isn't
+        //
+        internal static void HandleIfDatabaseUnavailableTypeException(Exception ex)
+        {
+            if (ex == null) return;
+            //System.Net.Sockets.SocketException (10054): An existing connection was forcibly closed by the remote host.
+            //System.Net.Sockets.SocketException (10061): No connection could be made because the target machine actively refused it.
+
+            do
+            {
+                if (ex.Message.Contains("SOMESTRING"))
+                {
+                    ServerGlobalOpsSettingsCache.DBAVAILABLE = true;
+                    return;
+                }
+                ex = ex.InnerException;
+                //ServerGlobalOpsSettingsCache.DBAVAILABLE = true;   
+            } while (ex != null);
+        }
+
         ///////////////////////////////////////////
         //Verify that database exists, if not, then create it
         //
@@ -514,7 +536,7 @@ namespace AyaNova.Util
                 await EraseTableAsync("apmitemoutsideservice", conn);
                 await EraseTableAsync("apmitem", conn);
                 await EraseTableAsync("apm", conn);//bugbug
-                //---
+                                                   //---
 
                 await EraseTableAsync("afileattachment", conn);
                 await EraseTableAsync("aevent", conn);