server/generator/CoreJobMetricsSnapshot.cs

@@ -0,0 +1,190 @@
+using System;
+using System.Linq;
+using System.Diagnostics;
+using Microsoft.Extensions.Logging;
+using Sockeye.Util;
+using Sockeye.Models;
+using Microsoft.EntityFrameworkCore;
+//using StackExchange.Profiling;
+
+
+namespace Sockeye.Biz
+{
+    /// <summary>
+    /// called by Generator to gather server metrics and insert in db
+    /// </summary>
+    internal static class CoreJobMetricsSnapshot
+    {
+        private static ILogger log = Sockeye.Util.ApplicationLogging.CreateLogger("CoreJobMetricsSnapshot");
+        private static TimeSpan tsDataRetention = new TimeSpan(365, 0, 0, 0, 0);//one year 
+        private static Process _process = Process.GetCurrentProcess();
+
+        private static TimeSpan _oldCPUTime = TimeSpan.Zero;
+        private static DateTime _lastMMSnapshot = DateTime.UtcNow.Subtract(new TimeSpan(1, 1, 1, 1, 1));//ensure it captures on a fresh boot;
+        private static DateTime _lastDDSnapshot = DateTime.UtcNow.Subtract(new TimeSpan(1, 1, 1, 1, 1));//ensure it captures on a fresh boot;
+
+        private static double _cpu = 0;
+
+#if (DEBUG)
+        private static TimeSpan tsMMFrequency = new TimeSpan(0, 5, 0);
+        private static TimeSpan tsDDFrequency = new TimeSpan(24, 0, 0);
+#else
+        private static TimeSpan tsMMFrequency = new TimeSpan(0, 5, 0);      
+        private static TimeSpan tsDDFrequency = new TimeSpan(24, 0, 0);//changed to 12 hours from 24 due to weird issue not gathering like it should diagnosis
+#endif
+        ////////////////////////////////////////////////////////////////////////////////////////////////
+        // 
+        //
+        public static void DoWork()
+        {
+
+
+            if (DateUtil.IsAfterDuration(_lastMMSnapshot, tsMMFrequency))
+            {
+                /////////////////////////////////////////////
+                //ONE MINUTE SNAPS
+                //
+                log.LogDebug("MM metrics snapshot");
+                var now = DateTime.UtcNow;
+                _process.Refresh();
+
+                //CPU
+                var cpuElapsedTime = now.Subtract(_lastMMSnapshot).TotalMilliseconds;
+                var newCPUTime = _process.TotalProcessorTime;
+                var elapsedCPU = (newCPUTime - _oldCPUTime).TotalMilliseconds;
+                _cpu = elapsedCPU * 100 / Environment.ProcessorCount / cpuElapsedTime;
+                _oldCPUTime = newCPUTime;
+
+                //MEMORY
+                // The memory occupied by objects.
+                var Allocated = GC.GetTotalMemory(false);//bigint
+
+                // The working set includes both shared and private data. The shared data includes the pages that contain all the 
+                // instructions that the process executes, including instructions in the process modules and the system libraries.
+                var WorkingSet = _process.WorkingSet64;//bigint
+
+                // The value returned by this property represents the current size of memory used by the process, in bytes, that 
+                // cannot be shared with other processes.
+                var PrivateBytes = _process.PrivateMemorySize64;//bigint
+
+
+                //NOTE: CPU percentage is *our* process cpu percentage over timeframe of last captured avg
+                //So it does *not* show the entire server cpu load, only for RAVEN.
+                //Overall, server stats need to be captured / viewed independently (digital ocean control panel for example or windows task manager)
+                var CPU = _cpu;// double precision
+                //In some cases the first snapshot taken after a reboot 
+                //is a huge number way beyond 100 which fucks up the charts
+                //likely due to the algorithm above and new values but not worth looking into atm
+                if (CPU > 100)
+                {
+                    CPU = 0;
+                }
+                using (AyContext ct = ServiceProviderProvider.DBContext)
+                {
+                    //write to db
+                    MetricMM mm = new MetricMM(Allocated, WorkingSet, PrivateBytes, CPU);
+                    ct.MetricMM.Add(mm);
+                    ct.SaveChanges();
+                }
+                _lastMMSnapshot = now;
+            }
+
+
+
+            /////////////////////////////////////////////
+            //ONCE A DAY SNAPS AND CLEANUP
+            //
+            if (DateUtil.IsAfterDuration(_lastDDSnapshot, tsDDFrequency))
+            {
+#if (DEBUG)
+                log.LogInformation($"DD metrics snapshot, _lastDDSnapshot:{_lastDDSnapshot}, tsDDFrequency:{tsDDFrequency}");
+#endif
+                log.LogDebug("DD metrics snapshot");
+
+                var now = DateTime.UtcNow;
+                //FILES ON DISK
+                var UtilFilesInfo = FileUtil.GetBackupFolderSizeInfo();
+                var AttachmentFilesInfo = FileUtil.GetAttachmentFolderSizeInfo();
+
+                //Available space
+                long UtilityFilesAvailableSpace = 0;
+                try
+                {
+                    UtilityFilesAvailableSpace = FileUtil.BackupFilesDriveAvailableSpace();
+                }
+                catch (Exception ex)
+                {
+                    log.LogError(ex, "Metrics::FileUtil::UtilityFilesDriveAvailableSpace error getting available space");
+                }
+
+                long AttachmentFilesAvailableSpace = 0;
+                try
+                {
+                    AttachmentFilesAvailableSpace = FileUtil.AttachmentFilesDriveAvailableSpace();
+                }
+                catch (Exception ex)
+                {
+                    log.LogError(ex, "Metrics::FileUtil::AttachmentFilesDriveAvailableSpace error getting available space");
+                }
+
+                using (AyContext ct = ServiceProviderProvider.DBContext)
+                {
+                    //DB total size          
+                    long DBTotalSize = 0;
+                    using (var command = ct.Database.GetDbConnection().CreateCommand())
+                    {
+                        command.CommandText = "select pg_database_size(current_database());";
+                        ct.Database.OpenConnection();
+                        using (var dr = command.ExecuteReader())
+                        {
+                            if (dr.HasRows)
+                            {
+                                DBTotalSize = dr.Read() ? dr.GetInt64(0) : 0;
+                            }
+                            ct.Database.CloseConnection();
+                        }
+                    }
+                    //write to db
+                    MetricDD dd = new MetricDD()
+                    {
+                        AttachmentFileSize = AttachmentFilesInfo.SizeWithChildren,
+                        AttachmentFileCount = AttachmentFilesInfo.FileCountWithChildren,
+                        AttachmentFilesAvailableSpace = AttachmentFilesAvailableSpace,
+                        UtilityFileSize = UtilFilesInfo.SizeWithChildren,
+                        UtilityFileCount = UtilFilesInfo.FileCountWithChildren,
+                        UtilityFilesAvailableSpace = UtilityFilesAvailableSpace,
+                        DBTotalSize = DBTotalSize
+                    };
+                    ct.MetricDD.Add(dd);
+                    ct.SaveChanges();
+                }
+
+
+                /////////////////////////////////
+                //CLEAR OLD ENTRIES
+                //
+
+                DateTime ClearDate = DateTime.UtcNow - tsDataRetention;
+                using (AyContext ct = ServiceProviderProvider.DBContext)
+                {
+                    ct.Database.ExecuteSqlInterpolated($"delete from ametricmm where t < {ClearDate.ToUniversalTime()}");
+                    ct.Database.ExecuteSqlInterpolated($"delete from ametricdd where t < {ClearDate.ToUniversalTime()}");
+                }
+                _lastDDSnapshot = now;
+            }
+
+
+
+
+            //----
+        }
+
+
+
+
+
+        /////////////////////////////////////////////////////////////////////
+
+    }//eoc
+}//eons
+