using System; using System.ComponentModel.DataAnnotations; using System.Linq; using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Routing; using Microsoft.AspNetCore.Authorization; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Newtonsoft.Json.Linq; using AyaNova.Models; using AyaNova.Api.ControllerHelpers; using AyaNova.Biz; using StackExchange.Profiling; namespace AyaNova.Api.Controllers { /// /// Server metrics /// [ApiController] [ApiVersion("8.0")] [Route("api/v{version:apiVersion}/server-metric")] [Authorize] public class ServerMetricsController : ControllerBase { private readonly AyContext ct; private readonly ILogger log; private readonly ApiServerState serverState; private const int DEFAULT_MAX_RECORDS = 400; private const long MB = (1024 * 1024); /// /// ctor /// /// /// /// public ServerMetricsController(AyContext dbcontext, ILogger logger, ApiServerState apiServerState) { ct = dbcontext; log = logger; serverState = apiServerState; } /// /// Get Memory and CPU server metrics for time period specified /// /// Start timestamp UTC /// End timestamp UTC /// Optional maximum records to return (downsampled). There is a 400 record maximum fixed default /// Snapshot of metrics [HttpGet("memcpu")] public async Task GetMemCPUMetrics([FromQuery, Required] DateTime? tsStart, [FromQuery, Required] DateTime? tsEnd, [FromQuery] int? maxRecords) { //Note: the date and times are nullable and required so that the regular modelstate code kicks in to ensure they are present if (serverState.IsClosed) return StatusCode(503, new ApiErrorResponse(serverState.ApiErrorCode, null, serverState.Reason)); if (!Authorized.HasReadFullRole(HttpContext.Items, AyaType.Metrics)) return StatusCode(403, new ApiNotAuthorizedResponse()); if (!ModelState.IsValid) return BadRequest(new ApiErrorResponse(ModelState)); maxRecords ??= DEFAULT_MAX_RECORDS; List MinuteMetrics = new List(); //touniversal is because the parameters are converted to local time here //but then sent to the query as local time as well and not universal time which is what it should be MinuteMetrics = await ct.MetricMM.AsNoTracking().Where(z => z.t >= ((DateTime)tsStart).ToUniversalTime() && z.t <= ((DateTime)tsEnd).ToUniversalTime()).OrderBy(z => z.t).ToListAsync(); var dsCPU = MinuteMetrics.Select(z => new Tuple(z.t.ToOADate(), z.CPU)).ToList(); dsCPU = Util.DataUtil.LargestTriangleThreeBuckets(dsCPU, (int)maxRecords) as List>; var dsAllocated = MinuteMetrics.Select(z => new Tuple(z.t.ToOADate(), z.Allocated)).ToList(); dsAllocated = Util.DataUtil.LargestTriangleThreeBuckets(dsAllocated, (int)maxRecords) as List>; var dsWorkingSet = MinuteMetrics.Select(z => new Tuple(z.t.ToOADate(), z.WorkingSet)).ToList(); dsWorkingSet = Util.DataUtil.LargestTriangleThreeBuckets(dsWorkingSet, (int)maxRecords) as List>; var dsPrivateBytes = MinuteMetrics.Select(z => new Tuple(z.t.ToOADate(), z.PrivateBytes)).ToList(); dsPrivateBytes = Util.DataUtil.LargestTriangleThreeBuckets(dsPrivateBytes, (int)maxRecords) as List>; var dsGen0 = MinuteMetrics.Select(z => new Tuple(z.t.ToOADate(), z.Gen0)).ToList(); dsGen0 = Util.DataUtil.LargestTriangleThreeBuckets(dsGen0, (int)maxRecords) as List>; var dsGen1 = MinuteMetrics.Select(z => new Tuple(z.t.ToOADate(), z.Gen1)).ToList(); dsGen1 = Util.DataUtil.LargestTriangleThreeBuckets(dsGen1, (int)maxRecords) as List>; var dsGen2 = MinuteMetrics.Select(z => new Tuple(z.t.ToOADate(), z.Gen2)).ToList(); dsGen2 = Util.DataUtil.LargestTriangleThreeBuckets(dsGen2, (int)maxRecords) as List>; var ret = new { cpu = dsCPU.Select(z => new MetricDouble(DateTime.FromOADate(z.Item1), z.Item2)).ToArray(), gen0 = dsGen0.Select(z => new MetricInt(DateTime.FromOADate(z.Item1), z.Item2)).ToArray(), gen1 = dsGen1.Select(z => new MetricInt(DateTime.FromOADate(z.Item1), z.Item2)).ToArray(), gen2 = dsGen2.Select(z => new MetricInt(DateTime.FromOADate(z.Item1), z.Item2)).ToArray(), allocated = dsAllocated.Select(z => new MetricLong(DateTime.FromOADate(z.Item1), z.Item2 / MB)).ToArray(), workingSet = dsWorkingSet.Select(z => new MetricLong(DateTime.FromOADate(z.Item1), z.Item2 / MB)).ToArray(), privateBytes = dsPrivateBytes.Select(z => new MetricLong(DateTime.FromOADate(z.Item1), z.Item2 / MB)).ToArray() }; await EventLogProcessor.LogEventToDatabaseAsync(new Event(UserIdFromContext.Id(HttpContext.Items), 0, AyaType.Metrics, AyaEvent.Retrieved), ct); return Ok(ApiOkResponse.Response(ret)); } /// /// Get storage server metrics for time period specified /// /// Start timestamp UTC /// End timestamp UTC /// Optional maximum records to return (downsampled). There is a 400 record maximum fixed default /// Snapshot of metrics [HttpGet("storage")] public async Task GetStorageMetrics([FromQuery, Required] DateTime? tsStart, [FromQuery, Required] DateTime? tsEnd, [FromQuery] int? maxRecords) { //Note: the date and times are nullable and required so that the regular modelstate code kicks in to ensure they are present if (serverState.IsClosed) return StatusCode(503, new ApiErrorResponse(serverState.ApiErrorCode, null, serverState.Reason)); if (!Authorized.HasReadFullRole(HttpContext.Items, AyaType.Metrics)) return StatusCode(403, new ApiNotAuthorizedResponse()); if (!ModelState.IsValid) return BadRequest(new ApiErrorResponse(ModelState)); maxRecords ??= DEFAULT_MAX_RECORDS; List DailyMetrics = new List(); //touniversal is because the parameters are converted to local time here //but then sent to the query as local time as well and not universal time which is what it should be DailyMetrics = await ct.MetricDD.AsNoTracking().Where(z => z.t >= ((DateTime)tsStart).ToUniversalTime() && z.t <= ((DateTime)tsEnd).ToUniversalTime()).OrderBy(z => z.t).ToListAsync(); var dsAttachmentFileCount = DailyMetrics.Select(z => new Tuple(z.t.ToOADate(), z.AttachmentFileCount)).ToList(); dsAttachmentFileCount = Util.DataUtil.LargestTriangleThreeBuckets(dsAttachmentFileCount, (int)maxRecords) as List>; var dsAttachmentFileSize = DailyMetrics.Select(z => new Tuple(z.t.ToOADate(), z.AttachmentFileSize)).ToList(); dsAttachmentFileSize = Util.DataUtil.LargestTriangleThreeBuckets(dsAttachmentFileSize, (int)maxRecords) as List>; var dsAttachmentFilesAvailableSpace = DailyMetrics.Select(z => new Tuple(z.t.ToOADate(), z.AttachmentFilesAvailableSpace)).ToList(); dsAttachmentFilesAvailableSpace = Util.DataUtil.LargestTriangleThreeBuckets(dsAttachmentFilesAvailableSpace, (int)maxRecords) as List>; var dsUtilityFileCount = DailyMetrics.Select(z => new Tuple(z.t.ToOADate(), z.UtilityFileCount)).ToList(); dsUtilityFileCount = Util.DataUtil.LargestTriangleThreeBuckets(dsUtilityFileCount, (int)maxRecords) as List>; var dsUtilityFileSize = DailyMetrics.Select(z => new Tuple(z.t.ToOADate(), z.UtilityFileSize)).ToList(); dsUtilityFileSize = Util.DataUtil.LargestTriangleThreeBuckets(dsUtilityFileSize, (int)maxRecords) as List>; var dsUtilityFilesAvailableSpace = DailyMetrics.Select(z => new Tuple(z.t.ToOADate(), z.UtilityFilesAvailableSpace)).ToList(); dsUtilityFilesAvailableSpace = Util.DataUtil.LargestTriangleThreeBuckets(dsUtilityFilesAvailableSpace, (int)maxRecords) as List>; var ret = new { attachmentFileCount = dsAttachmentFileCount.Select(z => new MetricLong(DateTime.FromOADate(z.Item1), z.Item2)).ToArray(), attachmentFileSize = dsAttachmentFileSize.Select(z => new MetricLong(DateTime.FromOADate(z.Item1), z.Item2 / MB)).ToArray(), attachmentFilesAvailableSpace = dsAttachmentFilesAvailableSpace.Select(z => new MetricLong(DateTime.FromOADate(z.Item1), z.Item2 / MB)).ToArray(), utilityFileCount = dsUtilityFileCount.Select(z => new MetricLong(DateTime.FromOADate(z.Item1), z.Item2)).ToArray(), utilityFileSize = dsUtilityFileSize.Select(z => new MetricLong(DateTime.FromOADate(z.Item1), z.Item2 / MB)).ToArray(), utilityFilesAvailableSpace = dsUtilityFilesAvailableSpace.Select(z => new MetricLong(DateTime.FromOADate(z.Item1), z.Item2 / MB)).ToArray() }; await EventLogProcessor.LogEventToDatabaseAsync(new Event(UserIdFromContext.Id(HttpContext.Items), 0, AyaType.Metrics, AyaEvent.Retrieved), ct); return Ok(ApiOkResponse.Response(ret)); } /// /// Get database related metrics for time period specified /// /// Start timestamp UTC /// End timestamp UTC /// Optional maximum records to return (downsampled). There is a 400 record maximum fixed default /// Snapshot of metrics [HttpGet("db")] public async Task GetDBMetrics([FromQuery, Required] DateTime? tsStart, [FromQuery, Required] DateTime? tsEnd, [FromQuery] int? maxRecords) { //Note: the date and times are nullable and required so that the regular modelstate code kicks in to ensure they are present if (serverState.IsClosed) return StatusCode(503, new ApiErrorResponse(serverState.ApiErrorCode, null, serverState.Reason)); if (!Authorized.HasReadFullRole(HttpContext.Items, AyaType.Metrics)) return StatusCode(403, new ApiNotAuthorizedResponse()); if (!ModelState.IsValid) return BadRequest(new ApiErrorResponse(ModelState)); maxRecords ??= DEFAULT_MAX_RECORDS; List DBMetrics = new List(); //touniversal is because the parameters are converted to local time here //but then sent to the query as local time as well and not universal time which is what it should be DBMetrics = await ct.MetricDD.AsNoTracking().Where(z => z.t >= ((DateTime)tsStart).ToUniversalTime() && z.t <= ((DateTime)tsEnd).ToUniversalTime()).OrderBy(z => z.t).ToListAsync(); var dsDBTotalSize = DBMetrics.Select(z => new Tuple(z.t.ToOADate(), z.DBTotalSize)).ToList(); dsDBTotalSize = Util.DataUtil.LargestTriangleThreeBuckets(dsDBTotalSize, (int)maxRecords) as List>; //table distribution, top 10 List TopTables = new List(); using (var command = ct.Database.GetDbConnection().CreateCommand()) { //top 10 tables by size command.CommandText = @"SELECT relname AS ""table_name"", pg_table_size(C.oid) AS ""table_size"" FROM pg_class C LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace) WHERE nspname NOT IN ('pg_catalog', 'information_schema') AND nspname !~ '^pg_toast' AND relkind IN ('r') ORDER BY pg_table_size(C.oid) DESC;"; ct.Database.OpenConnection(); using (var dr = command.ExecuteReader()) { if (dr.HasRows) { while (dr.Read()) { long tableSize = dr.GetInt64(1); string tableName = dr.GetString(0); if (tableSize > 0) { tableSize = tableSize / MB; } TopTables.Add(new MetricNameLongValue() { name = tableName, value = tableSize }); } } ct.Database.CloseConnection(); } } //trim out tables we don't want here TopTables = TopTables.Where(z => z.value > 0).OrderByDescending(z => z.value).ToList(); long DBTotalSize = 0; using (var command = ct.Database.GetDbConnection().CreateCommand()) { command.CommandText = "select pg_database_size(current_database());"; ct.Database.OpenConnection(); using (var dr = command.ExecuteReader()) { if (dr.HasRows) { DBTotalSize = dr.Read() ? (dr.GetInt64(0) / MB) : 0; } ct.Database.CloseConnection(); } } long ttSize = 0; foreach (MetricNameLongValue tt in TopTables) { ttSize += tt.value; } TopTables.Add(new MetricNameLongValue() { name = "other", value = DBTotalSize - ttSize }); var ret = new { TopTables = TopTables, totalSize = dsDBTotalSize.Select(z => new MetricLong(DateTime.FromOADate(z.Item1), z.Item2 / MB)).ToArray() }; await EventLogProcessor.LogEventToDatabaseAsync(new Event(UserIdFromContext.Id(HttpContext.Items), 0, AyaType.Metrics, AyaEvent.Retrieved), ct); return Ok(ApiOkResponse.Response(ret)); } //------------ public class MetricLong { public DateTime x { get; set; } public long y { get; set; } public MetricLong(DateTime px, double py) { x = px; y = (long)py; } } public class MetricInt { public DateTime x { get; set; } public int y { get; set; } public MetricInt(DateTime px, double py) { x = px; y = (int)py; } } public class MetricDouble { public DateTime x { get; set; } public double y { get; set; } public MetricDouble(DateTime px, double py) { x = px; y = py; } } public class MetricNameLongValue { public string name { get; set; } public long value { get; set; } } //---------- } }