diff --git a/server/AyaNova/biz/JobsBiz.cs b/server/AyaNova/biz/JobsBiz.cs index e1529813..22176957 100644 --- a/server/AyaNova/biz/JobsBiz.cs +++ b/server/AyaNova/biz/JobsBiz.cs @@ -186,6 +186,14 @@ namespace AyaNova.Biz return; } + //Do not process if there is no db, everything relies on it below here + if (!ServerGlobalOpsSettingsCache.DBAVAILABLE) + { + //This will set dbavailable flag if it becomes available + DbUtil.CheckDatabaseServerAvailable(); + return; + } + ActivelyProcessing = true; log.LogTrace("Processing internal jobs"); try @@ -307,6 +315,7 @@ namespace AyaNova.Biz { var msg = "Server::ProcessJobsAsync unexpected error during processing"; log.LogError(ex, msg); + DbUtil.HandleIfDatabaseUnavailableTypeException(ex); await NotifyEventHelper.AddOpsProblemEvent(msg, ex); } finally diff --git a/server/AyaNova/biz/NotifyEventHelper.cs b/server/AyaNova/biz/NotifyEventHelper.cs index e0aecfc3..f8e0ce9c 100644 --- a/server/AyaNova/biz/NotifyEventHelper.cs +++ b/server/AyaNova/biz/NotifyEventHelper.cs @@ -395,6 +395,7 @@ namespace AyaNova.Biz catch (Exception ex) { log.LogError(ex, $"Error adding general notify event [type:{eventType}, userId:{userId}, message:{message}]"); + DbUtil.HandleIfDatabaseUnavailableTypeException(ex); } }//eom diff --git a/server/AyaNova/biz/PMBiz.cs b/server/AyaNova/biz/PMBiz.cs index 7e0d2ad5..c051f506 100644 --- a/server/AyaNova/biz/PMBiz.cs +++ b/server/AyaNova/biz/PMBiz.cs @@ -5211,6 +5211,7 @@ namespace AyaNova.Biz catch (Exception ex) { log.LogError(ex, $"error generating Work order from PM {p.Serial}"); + DbUtil.HandleIfDatabaseUnavailableTypeException(ex); await NotifyEventHelper.AddGeneralNotifyEvent(AyaType.PM, p.Id, NotifyEventType.PMGenerationFailed, $"Error generating Work order from PM {p.Serial}", "Preventive Maintenance", ex); if (!KeepOnWorking(log)) return; continue; diff --git a/server/AyaNova/generator/CoreJobCustomerNotify.cs b/server/AyaNova/generator/CoreJobCustomerNotify.cs index 924fd3f8..a9df97e3 100644 --- a/server/AyaNova/generator/CoreJobCustomerNotify.cs +++ b/server/AyaNova/generator/CoreJobCustomerNotify.cs @@ -100,6 +100,7 @@ namespace AyaNova.Biz catch (Exception ex) { log.LogError(ex, $"Error processing customer notification event"); + DbUtil.HandleIfDatabaseUnavailableTypeException(ex); } finally { diff --git a/server/AyaNova/generator/CoreJobNotify.cs b/server/AyaNova/generator/CoreJobNotify.cs index 3daf2d93..627a8290 100644 --- a/server/AyaNova/generator/CoreJobNotify.cs +++ b/server/AyaNova/generator/CoreJobNotify.cs @@ -141,6 +141,7 @@ namespace AyaNova.Biz catch (Exception ex) { log.LogError(ex, $"Error processing notification event"); + DbUtil.HandleIfDatabaseUnavailableTypeException(ex); } finally { diff --git a/server/AyaNova/generator/CoreJobPMGenerate.cs b/server/AyaNova/generator/CoreJobPMGenerate.cs index 83d2e154..b3990c21 100644 --- a/server/AyaNova/generator/CoreJobPMGenerate.cs +++ b/server/AyaNova/generator/CoreJobPMGenerate.cs @@ -56,7 +56,9 @@ namespace AyaNova.Biz catch (Exception ex) { log.LogError(ex, $"Error processing PMGenerate "); - await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.PMGenerationFailed, "PM Generate failure","Preventive Maintenance", ex); + DbUtil.HandleIfDatabaseUnavailableTypeException(ex); + await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.PMGenerationFailed, "PM Generate failure", "Preventive Maintenance", ex); + } finally { diff --git a/server/AyaNova/generator/CoreJobPMInventoryCheck.cs b/server/AyaNova/generator/CoreJobPMInventoryCheck.cs index a949224b..35b984ca 100644 --- a/server/AyaNova/generator/CoreJobPMInventoryCheck.cs +++ b/server/AyaNova/generator/CoreJobPMInventoryCheck.cs @@ -58,6 +58,7 @@ namespace AyaNova.Biz catch (Exception ex) { log.LogError(ex, $"Error processing CoreJobPMInventoryCheck "); + DbUtil.HandleIfDatabaseUnavailableTypeException(ex); await NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType.PMGenerationFailed, "PM Inventory check failure", "Preventive Maintenance", ex); } finally diff --git a/server/AyaNova/util/DbUtil.cs b/server/AyaNova/util/DbUtil.cs index fb63f36c..616a5620 100644 --- a/server/AyaNova/util/DbUtil.cs +++ b/server/AyaNova/util/DbUtil.cs @@ -232,6 +232,28 @@ namespace AyaNova.Util private static TimeSpan CHECK_IF_DB_SERVER_AVAILABLE_DELAY = new TimeSpan(0, 0, 10); + /////////////////////////////////////////////////////////// + // Check if exception means db server is unavailable + // if so, flag global flag indicating it isn't + // + internal static void HandleIfDatabaseUnavailableTypeException(Exception ex) + { + if (ex == null) return; + //System.Net.Sockets.SocketException (10054): An existing connection was forcibly closed by the remote host. + //System.Net.Sockets.SocketException (10061): No connection could be made because the target machine actively refused it. + + do + { + if (ex.Message.Contains("SOMESTRING")) + { + ServerGlobalOpsSettingsCache.DBAVAILABLE = true; + return; + } + ex = ex.InnerException; + //ServerGlobalOpsSettingsCache.DBAVAILABLE = true; + } while (ex != null); + } + /////////////////////////////////////////// //Verify that database exists, if not, then create it // @@ -514,7 +536,7 @@ namespace AyaNova.Util await EraseTableAsync("apmitemoutsideservice", conn); await EraseTableAsync("apmitem", conn); await EraseTableAsync("apm", conn);//bugbug - //--- + //--- await EraseTableAsync("afileattachment", conn); await EraseTableAsync("aevent", conn);