using System; using System.Linq; using System.Diagnostics; using Microsoft.Extensions.Logging; using AyaNova.Util; using AyaNova.Models; using Microsoft.EntityFrameworkCore; using StackExchange.Profiling; namespace AyaNova.Biz { /// /// called by Generator to gather server metrics and insert in db /// internal static class CoreJobMetricsSnapshot { private static ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("CoreJobMetricsSnapshot"); private static TimeSpan tsDataRetention = new TimeSpan(365, 0, 0, 0, 0);//one year private static Process _process = Process.GetCurrentProcess(); private static TimeSpan _oldCPUTime = TimeSpan.Zero; private static DateTime _lastMMSnapshot = DateTime.UtcNow; private static DateTime _lastHHSnapshot = DateTime.UtcNow; private static DateTime _lastDDSnapshot = DateTime.UtcNow; // private static DateTime _lastRpsTime = DateTime.UtcNow; private static double _cpu = 0; #if(DEBUG) private static TimeSpan tsOneMinute = new TimeSpan(0, 0, 10); private static TimeSpan tsOneHour = new TimeSpan(0, 1, 0); private static TimeSpan ts24Hours = new TimeSpan(0, 1, 0); #else private static TimeSpan tsOneMinute = new TimeSpan(0, 1, 0); private static TimeSpan tsOneHour = new TimeSpan(1, 0, 0); private static TimeSpan ts24Hours = new TimeSpan(24, 0, 0); #endif //////////////////////////////////////////////////////////////////////////////////////////////// // DoAsync // public static void DoJob() { if (DateUtil.IsAfterDuration(_lastMMSnapshot, tsOneMinute)) { ///////////////////////////////////////////// //ONE MINUTE SNAPS // log.LogTrace("MM metrics snapshot"); var now = DateTime.UtcNow; _process.Refresh(); //CPU var cpuElapsedTime = now.Subtract(_lastMMSnapshot).TotalMilliseconds; var newCPUTime = _process.TotalProcessorTime; var elapsedCPU = (newCPUTime - _oldCPUTime).TotalMilliseconds; _cpu = elapsedCPU * 100 / Environment.ProcessorCount / cpuElapsedTime; _oldCPUTime = newCPUTime; //MEMORY // The memory occupied by objects. var Allocated = GC.GetTotalMemory(false);//bigint // The working set includes both shared and private data. The shared data includes the pages that contain all the // instructions that the process executes, including instructions in the process modules and the system libraries. var WorkingSet = _process.WorkingSet64;//bigint // The value returned by this property represents the current size of memory used by the process, in bytes, that // cannot be shared with other processes. var PrivateBytes = _process.PrivateMemorySize64;//bigint // The number of generation 0 collections var Gen0 = GC.CollectionCount(0);//integer // The number of generation 1 collections var Gen1 = GC.CollectionCount(1);//integer // The number of generation 2 collections var Gen2 = GC.CollectionCount(2);//integer //NOTE: CPU percentage is *our* process cpu percentage over timeframe of last captured avg //So it does *not* show the entire server cpu load, only for RAVEN, server stats //need to be captured / viewed independently (digital ocean control panel for example or windows task manager) var CPU = _cpu;// double precision //System.Diagnostics.Debug.WriteLine($"MM Snapshot, cpu: {CPU}"); using (AyContext ct = ServiceProviderProvider.DBContext) { //write to db MetricMM mm = new MetricMM(Allocated, WorkingSet, PrivateBytes, Gen0, Gen1, Gen2, CPU); ct.MetricMM.Add(mm); ct.SaveChanges(); //System.Diagnostics.Debug.WriteLine("MM SAVED"); } _lastMMSnapshot = now; //TEST var profiler = MiniProfiler.StartNew("My Profiler Name"); if (profiler != null) { var Options = profiler.Options; var guids = Options.Storage.List(100); // var lastId = context.Request["last-id"]; // if (!lastId.IsNullOrWhiteSpace() && Guid.TryParse(lastId, out var lastGuid)) // { // guids = guids.TakeWhile(g => g != lastGuid); // } var ministats = guids.Reverse() .Select(g => Options.Storage.Load(g)) .Where(p => p != null) .Select(p => new { p.Id, p.Name, p.ClientTimings, p.Started, p.HasUserViewed, p.MachineName, p.User, p.DurationMilliseconds }).ToList(); if(ministats.Count>0){ var v=ministats.Count; } } } ///////////////////////////////////////////// //EVERY HOUR SNAPS // if (DateUtil.IsAfterDuration(_lastHHSnapshot, tsOneHour)) { var now = DateTime.UtcNow; log.LogTrace("HH metrics snapshot"); //RECORDS IN TABLE // //Only do this once per hour // log.LogTrace("Counting table records"); // //Get a count of important tables in db // List allTableNames = await DbUtil.GetAllTablenamesAsync(); // //Skip some tables as they are internal and / or only ever have one record // List skipTableNames = new List(); // skipTableNames.Add("alicense"); // skipTableNames.Add("aschemaversion"); // foreach (string table in allTableNames) // { // if (!skipTableNames.Contains(table)) // { // //var tags = new MetricTags("TableTagKey", table); // // metrics.Measure.Gauge.SetValue(MetricsRegistry.DBRecordsGauge, tags, await DbUtil.CountOfRecordsAsync(table)); // } // } //JOB COUNTS (DEAD, RUNNING, COMPLETED, SLEEPING) // foreach (JobStatus stat in Enum.GetValues(typeof(JobStatus))) // { // // var jobtag = new MetricTags("JobStatus", stat.ToString()); // // metrics.Measure.Gauge.SetValue(MetricsRegistry.JobsGauge, jobtag, await JobsBiz.GetCountForJobStatusAsync(ct, stat)); // } _lastHHSnapshot = now; } ///////////////////////////////////////////// //ONCE A DAY SNAPS AND CLEANUP // if (DateUtil.IsAfterDuration(_lastDDSnapshot, ts24Hours)) { log.LogTrace("DD metrics snapshot"); var now = DateTime.UtcNow; //FILES ON DISK var UtilFilesInfo = FileUtil.GetUtilityFolderSizeInfo(); var AttachmentFilesInfo = FileUtil.GetAttachmentFolderSizeInfo(); //Available space var UtilityFilesAvailableSpace = FileUtil.UtilityFilesDriveAvailableSpace(); var AttachmentFilesAvailableSpace = FileUtil.AttachmentFilesDriveAvailableSpace(); using (AyContext ct = ServiceProviderProvider.DBContext) { //write to db MetricDD dd = new MetricDD() { AttachmentFileSize = AttachmentFilesInfo.SizeWithChildren, AttachmentFileCount = AttachmentFilesInfo.FileCountWithChildren, AttachmentFilesAvailableSpace = AttachmentFilesAvailableSpace, UtilityFileSize = UtilFilesInfo.SizeWithChildren, UtilityFileCount = UtilFilesInfo.FileCountWithChildren, UtilityFilesAvailableSpace = UtilityFilesAvailableSpace }; ct.MetricDD.Add(dd); ct.SaveChanges(); } ///////////////////////////////// //CLEAR OLD ENTRIES // DateTime ClearDate = DateTime.UtcNow - tsDataRetention; using (AyContext ct = ServiceProviderProvider.DBContext) { // ct.Database.ExecuteSqlInterpolated($"delete from ametricmm where t < {ClearDate.ToUniversalTime()}"); } _lastDDSnapshot = now; } //---- } ///////////////////////////////////////////////////////////////////// }//eoc }//eons