sockeye/server/generator/CoreJobMetricsSnapshot.cs

using System;
using System.Linq;
using System.Diagnostics;
using Microsoft.Extensions.Logging;
using Sockeye.Util;
using Sockeye.Models;
using Microsoft.EntityFrameworkCore;
//using StackExchange.Profiling;


namespace Sockeye.Biz
{
    /// <summary>
    /// called by Generator to gather server metrics and insert in db
    /// </summary>
    internal static class CoreJobMetricsSnapshot
    {
        private static ILogger log = Sockeye.Util.ApplicationLogging.CreateLogger("CoreJobMetricsSnapshot");
        private static TimeSpan tsDataRetention = new TimeSpan(365, 0, 0, 0, 0);//one year 
        private static Process _process = Process.GetCurrentProcess();

        private static TimeSpan _oldCPUTime = TimeSpan.Zero;
        private static DateTime _lastMMSnapshot = DateTime.UtcNow.Subtract(new TimeSpan(1, 1, 1, 1, 1));//ensure it captures on a fresh boot;
        private static DateTime _lastDDSnapshot = DateTime.UtcNow.Subtract(new TimeSpan(1, 1, 1, 1, 1));//ensure it captures on a fresh boot;

        private static double _cpu = 0;

#if (DEBUG)
        private static TimeSpan tsMMFrequency = new TimeSpan(0, 5, 0);
        private static TimeSpan tsDDFrequency = new TimeSpan(24, 0, 0);
#else
        private static TimeSpan tsMMFrequency = new TimeSpan(0, 5, 0);      
        private static TimeSpan tsDDFrequency = new TimeSpan(24, 0, 0);//changed to 12 hours from 24 due to weird issue not gathering like it should diagnosis
#endif
        ////////////////////////////////////////////////////////////////////////////////////////////////
        // 
        //
        public static void DoWork()
        {


            if (DateUtil.IsAfterDuration(_lastMMSnapshot, tsMMFrequency))
            {
                /////////////////////////////////////////////
                //ONE MINUTE SNAPS
                //
                log.LogDebug("MM metrics snapshot");
                var now = DateTime.UtcNow;
                _process.Refresh();

                //CPU
                var cpuElapsedTime = now.Subtract(_lastMMSnapshot).TotalMilliseconds;
                var newCPUTime = _process.TotalProcessorTime;
                var elapsedCPU = (newCPUTime - _oldCPUTime).TotalMilliseconds;
                _cpu = elapsedCPU * 100 / Environment.ProcessorCount / cpuElapsedTime;
                _oldCPUTime = newCPUTime;

                //MEMORY
                // The memory occupied by objects.
                var Allocated = GC.GetTotalMemory(false);//bigint

                // The working set includes both shared and private data. The shared data includes the pages that contain all the 
                // instructions that the process executes, including instructions in the process modules and the system libraries.
                var WorkingSet = _process.WorkingSet64;//bigint

                // The value returned by this property represents the current size of memory used by the process, in bytes, that 
                // cannot be shared with other processes.
                var PrivateBytes = _process.PrivateMemorySize64;//bigint


                //NOTE: CPU percentage is *our* process cpu percentage over timeframe of last captured avg
                //So it does *not* show the entire server cpu load, only for RAVEN.
                //Overall, server stats need to be captured / viewed independently (digital ocean control panel for example or windows task manager)
                var CPU = _cpu;// double precision
                //In some cases the first snapshot taken after a reboot 
                //is a huge number way beyond 100 which fucks up the charts
                //likely due to the algorithm above and new values but not worth looking into atm
                if (CPU > 100)
                {
                    CPU = 0;
                }
                using (AyContext ct = ServiceProviderProvider.DBContext)
                {
                    //write to db
                    MetricMM mm = new MetricMM(Allocated, WorkingSet, PrivateBytes, CPU);
                    ct.MetricMM.Add(mm);
                    ct.SaveChanges();
                }
                _lastMMSnapshot = now;
            }



            /////////////////////////////////////////////
            //ONCE A DAY SNAPS AND CLEANUP
            //
            if (DateUtil.IsAfterDuration(_lastDDSnapshot, tsDDFrequency))
            {
#if (DEBUG)
                log.LogInformation($"DD metrics snapshot, _lastDDSnapshot:{_lastDDSnapshot}, tsDDFrequency:{tsDDFrequency}");
#endif
                log.LogDebug("DD metrics snapshot");

                var now = DateTime.UtcNow;
                //FILES ON DISK
                var UtilFilesInfo = FileUtil.GetBackupFolderSizeInfo();
                var AttachmentFilesInfo = FileUtil.GetAttachmentFolderSizeInfo();

                //Available space
                long UtilityFilesAvailableSpace = 0;
                try
                {
                    UtilityFilesAvailableSpace = FileUtil.BackupFilesDriveAvailableSpace();
                }
                catch (Exception ex)
                {
                    log.LogError(ex, "Metrics::FileUtil::UtilityFilesDriveAvailableSpace error getting available space");
                }

                long AttachmentFilesAvailableSpace = 0;
                try
                {
                    AttachmentFilesAvailableSpace = FileUtil.AttachmentFilesDriveAvailableSpace();
                }
                catch (Exception ex)
                {
                    log.LogError(ex, "Metrics::FileUtil::AttachmentFilesDriveAvailableSpace error getting available space");
                }

                using (AyContext ct = ServiceProviderProvider.DBContext)
                {
                    //DB total size          
                    long DBTotalSize = 0;
                    using (var command = ct.Database.GetDbConnection().CreateCommand())
                    {
                        command.CommandText = "select pg_database_size(current_database());";
                        ct.Database.OpenConnection();
                        using (var dr = command.ExecuteReader())
                        {
                            if (dr.HasRows)
                            {
                                DBTotalSize = dr.Read() ? dr.GetInt64(0) : 0;
                            }
                            ct.Database.CloseConnection();
                        }
                    }
                    //write to db
                    MetricDD dd = new MetricDD()
                    {
                        AttachmentFileSize = AttachmentFilesInfo.SizeWithChildren,
                        AttachmentFileCount = AttachmentFilesInfo.FileCountWithChildren,
                        AttachmentFilesAvailableSpace = AttachmentFilesAvailableSpace,
                        UtilityFileSize = UtilFilesInfo.SizeWithChildren,
                        UtilityFileCount = UtilFilesInfo.FileCountWithChildren,
                        UtilityFilesAvailableSpace = UtilityFilesAvailableSpace,
                        DBTotalSize = DBTotalSize
                    };
                    ct.MetricDD.Add(dd);
                    ct.SaveChanges();
                }


                /////////////////////////////////
                //CLEAR OLD ENTRIES
                //

                DateTime ClearDate = DateTime.UtcNow - tsDataRetention;
                using (AyContext ct = ServiceProviderProvider.DBContext)
                {
                    ct.Database.ExecuteSqlInterpolated($"delete from ametricmm where t < {ClearDate.ToUniversalTime()}");
                    ct.Database.ExecuteSqlInterpolated($"delete from ametricdd where t < {ClearDate.ToUniversalTime()}");
                }
                _lastDDSnapshot = now;
            }




            //----
        }





        /////////////////////////////////////////////////////////////////////

    }//eoc
}//eons