diff --git a/.vscode/launch.json b/.vscode/launch.json index 95c9b301..96f419bb 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -40,8 +40,8 @@ "env": { "ASPNETCORE_ENVIRONMENT": "Development", "AYANOVA_JWT_SECRET": "UNLICENSED5G*QQJ8#bQ7$Xr_@sXfHq4", - "AYANOVA_LOG_LEVEL": "Info", - //"AYANOVA_LOG_LEVEL": "Debug", + //"AYANOVA_LOG_LEVEL": "Info", + "AYANOVA_LOG_LEVEL": "Debug", "AYANOVA_DEFAULT_TRANSLATION": "en", //TRANSLATION MUST BE en for Integration TESTING //"AYANOVA_PERMANENTLY_ERASE_DATABASE": "true", diff --git a/devdocs/todo.txt b/devdocs/todo.txt index 10d581c1..107e5e4c 100644 --- a/devdocs/todo.txt +++ b/devdocs/todo.txt @@ -1,6 +1,48 @@ {"login": "manager","password": "l3tm3in"} + + +todo: MAYBE a datetime kind of parameter is interpreted as LOCAL by the server on a route + so what the fuck + sent from client http://localhost:7575/api/v8.0/server-metric?maxRecords=400&tsStart=2020-05-27T13:07:39.068Z&tsEnd=2020-05-27T19:07:40.068Z + At server route: tsStart: {2020-05-27 6:07:39 AM} tsEnd: {2020-05-27 12:07:40 PM} Kind=local for both + so actually that's all good so far, how about the query itself? + + 2020-05-27 12:13:38.6439|DEBUG|Microsoft.EntityFrameworkCore.Database.Command|Executing DbCommand [Parameters=[@__tsStart_0='2020-05-27T06:13:32' (DbType = DateTime), @__tsEnd_1='2020-05-27T12:13:33' (DbType = DateTime)], CommandType='Text', CommandTimeout='30'] +SELECT a.t, a.allocated, a.cpu, a.gen0, a.gen1, a.gen2, a.privatebytes, a.workingset +FROM ametricmm AS a +WHERE (a.t >= @__tsStart_0) AND (a.t <= @__tsEnd_1) +ORDER BY a.t DESC +2020-05-27 12:13:38.8848|INFO|Microsoft.EntityFrameworkCore.Database.Command|Executed DbCommand (241ms) [Parameters=[@__tsStart_0='2020-05-27T06:13:32' (DbType = DateTime), @__tsEnd_1='2020-05-27T12:13:33' (DbType = DateTime)], CommandType='Text', CommandTimeout='30'] +SELECT a.t, a.allocated, a.cpu, a.gen0, a.gen1, a.gen2, a.privatebytes, a.workingset +FROM ametricmm AS a +WHERE (a.t >= @__tsStart_0) AND (a.t <= @__tsEnd_1) +ORDER BY a.t DESC + + Well, shit. The query is sending local time to the db server, so any query that sends to teh server needs teh values converted to utc first + This is malarky pure and simple! + :) + +If change parameters to DateTimeOffset from DateTime and rerun exact steps as above: +2020-05-27 12:21:49.8196|INFO|Microsoft.EntityFrameworkCore.Database.Command|Executed DbCommand (21ms) [Parameters=[@__tsStart_0='2020-05-27T13:21:48.4840000+00:00', @__tsEnd_1='2020-05-27T19:21:49.4840000+00:00'], CommandType='Text', CommandTimeout='30'] +SELECT a.t, a.allocated, a.cpu, a.gen0, a.gen1, a.gen2, a.privatebytes, a.workingset +FROM ametricmm AS a +WHERE (CAST(a.t AS timestamp with time zone) >= @__tsStart_0) AND (CAST(a.t AS timestamp with time zone) <= @__tsEnd_1) +ORDER BY a.t + +* Now it properly is sending the value to the db in UTC as it stays IN utc from input to query +One thing though is it's casting the query to a timestamp with timezone which is weird but maybe because the column is defined as timestamp witout timezone + +going to try + changing the model to datetimeoffset + change database schema for mmetric to with timezone + no, actually for me it's better without because it will never try to interpret or change it, it will just return and accept it as is and + that's what I want if I'm only dealing in UTC + rerun again and see if it's casting still + + + todo: leave running check it does overnight backup properly todo: add backup turn off setting diff --git a/server/AyaNova/Controllers/ServerMetricsController.cs b/server/AyaNova/Controllers/ServerMetricsController.cs index b7ead9f1..8a62ae02 100644 --- a/server/AyaNova/Controllers/ServerMetricsController.cs +++ b/server/AyaNova/Controllers/ServerMetricsController.cs @@ -57,7 +57,7 @@ namespace AyaNova.Api.Controllers /// Optional maximum records to return. If there are more records for the time period selected than this value the result will be downsampled. There is a 400 record maximum fixed default /// Snapshot of metrics [HttpGet] - public async Task GetMetrics([FromQuery] DateTime tsStart, [FromQuery] DateTime tsEnd, [FromQuery] int? maxRecords) + public async Task GetMetrics([FromQuery] DateTimeOffset tsStart, [FromQuery] DateTimeOffset tsEnd, [FromQuery] int? maxRecords) { if (serverState.IsClosed) return StatusCode(503, new ApiErrorResponse(serverState.ApiErrorCode, null, serverState.Reason)); @@ -72,6 +72,7 @@ namespace AyaNova.Api.Controllers List MinuteMetrics = new List(); MinuteMetrics = await ct.MetricMM.AsNoTracking().Where(z => z.t >= tsStart && z.t <= tsEnd).OrderBy(z => z.t).ToListAsync(); +//var v=await ct.MetricMM.AsNoTracking().Where(z => z.t >= tsStart.ToUniversalTime() && z.t <= tsEnd.ToUniversalTime()).OrderByDescending(z => z.t).ToListAsync(); //Log await EventLogProcessor.LogEventToDatabaseAsync(new Event(UserIdFromContext.Id(HttpContext.Items), 0, AyaType.Metrics, AyaEvent.Retrieved), ct); @@ -82,25 +83,47 @@ namespace AyaNova.Api.Controllers { //yes, so need to return individual labels and downsampled data as they wont' sync anymore - var dsCPU = MinuteMetrics.Select(z => new Tuple(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.CPU)).ToList(); + // var dsCPU = MinuteMetrics.Select(z => new Tuple(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.CPU)).ToList(); + // dsCPU = Util.DataUtil.LargestTriangleThreeBuckets(dsCPU, (int)maxRecords) as List>; + + // var dsAllocated = MinuteMetrics.Select(z => new Tuple(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.Allocated)).ToList(); + // dsAllocated = Util.DataUtil.LargestTriangleThreeBuckets(dsAllocated, (int)maxRecords) as List>; + + // var dsWorkingSet = MinuteMetrics.Select(z => new Tuple(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.WorkingSet)).ToList(); + // dsWorkingSet = Util.DataUtil.LargestTriangleThreeBuckets(dsWorkingSet, (int)maxRecords) as List>; + + // var dsPrivateBytes = MinuteMetrics.Select(z => new Tuple(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.PrivateBytes)).ToList(); + // dsPrivateBytes = Util.DataUtil.LargestTriangleThreeBuckets(dsPrivateBytes, (int)maxRecords) as List>; + + // var dsGen0 = MinuteMetrics.Select(z => new Tuple(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.Gen0)).ToList(); + // dsGen0 = Util.DataUtil.LargestTriangleThreeBuckets(dsGen0, (int)maxRecords) as List>; + + // var dsGen1 = MinuteMetrics.Select(z => new Tuple(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.Gen1)).ToList(); + // dsGen1 = Util.DataUtil.LargestTriangleThreeBuckets(dsGen1, (int)maxRecords) as List>; + + // var dsGen2 = MinuteMetrics.Select(z => new Tuple(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.Gen2)).ToList(); + // dsGen2 = Util.DataUtil.LargestTriangleThreeBuckets(dsGen2, (int)maxRecords) as List>; + + + var dsCPU = MinuteMetrics.Select(z => new Tuple(z.t.ToUnixTimeSeconds(), z.CPU)).ToList(); dsCPU = Util.DataUtil.LargestTriangleThreeBuckets(dsCPU, (int)maxRecords) as List>; - var dsAllocated = MinuteMetrics.Select(z => new Tuple(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.Allocated)).ToList(); + var dsAllocated = MinuteMetrics.Select(z => new Tuple(z.t.ToUnixTimeSeconds(), z.Allocated)).ToList(); dsAllocated = Util.DataUtil.LargestTriangleThreeBuckets(dsAllocated, (int)maxRecords) as List>; - var dsWorkingSet = MinuteMetrics.Select(z => new Tuple(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.WorkingSet)).ToList(); + var dsWorkingSet = MinuteMetrics.Select(z => new Tuple(z.t.ToUnixTimeSeconds(), z.WorkingSet)).ToList(); dsWorkingSet = Util.DataUtil.LargestTriangleThreeBuckets(dsWorkingSet, (int)maxRecords) as List>; - var dsPrivateBytes = MinuteMetrics.Select(z => new Tuple(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.PrivateBytes)).ToList(); + var dsPrivateBytes = MinuteMetrics.Select(z => new Tuple(z.t.ToUnixTimeSeconds(), z.PrivateBytes)).ToList(); dsPrivateBytes = Util.DataUtil.LargestTriangleThreeBuckets(dsPrivateBytes, (int)maxRecords) as List>; - var dsGen0 = MinuteMetrics.Select(z => new Tuple(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.Gen0)).ToList(); + var dsGen0 = MinuteMetrics.Select(z => new Tuple(z.t.ToUnixTimeSeconds(), z.Gen0)).ToList(); dsGen0 = Util.DataUtil.LargestTriangleThreeBuckets(dsGen0, (int)maxRecords) as List>; - var dsGen1 = MinuteMetrics.Select(z => new Tuple(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.Gen1)).ToList(); + var dsGen1 = MinuteMetrics.Select(z => new Tuple(z.t.ToUnixTimeSeconds(), z.Gen1)).ToList(); dsGen1 = Util.DataUtil.LargestTriangleThreeBuckets(dsGen1, (int)maxRecords) as List>; - var dsGen2 = MinuteMetrics.Select(z => new Tuple(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.Gen2)).ToList(); + var dsGen2 = MinuteMetrics.Select(z => new Tuple(z.t.ToUnixTimeSeconds(), z.Gen2)).ToList(); dsGen2 = Util.DataUtil.LargestTriangleThreeBuckets(dsGen2, (int)maxRecords) as List>; diff --git a/server/AyaNova/Startup.cs b/server/AyaNova/Startup.cs index 76a8904e..72cfdf67 100644 --- a/server/AyaNova/Startup.cs +++ b/server/AyaNova/Startup.cs @@ -134,7 +134,7 @@ namespace AyaNova bool LOG_SENSITIVE_DATA = false; #if (DEBUG) - LOG_SENSITIVE_DATA = false;//############################################################################ + LOG_SENSITIVE_DATA = true;//############################################################################ #endif diff --git a/server/AyaNova/generator/CoreJobMetricsSnapshot.cs b/server/AyaNova/generator/CoreJobMetricsSnapshot.cs index 9e07825f..2efe8e78 100644 --- a/server/AyaNova/generator/CoreJobMetricsSnapshot.cs +++ b/server/AyaNova/generator/CoreJobMetricsSnapshot.cs @@ -24,7 +24,7 @@ namespace AyaNova.Biz private static double _cpu = 0; #if(DEBUG) - private static TimeSpan tsOneMinute = new TimeSpan(0, 1,0); + private static TimeSpan tsOneMinute = new TimeSpan(0, 0,10); #else private static TimeSpan tsOneMinute = new TimeSpan(0, 1, 0); #endif diff --git a/server/AyaNova/models/MetricMM.cs b/server/AyaNova/models/MetricMM.cs index 8093d43c..719030cf 100644 --- a/server/AyaNova/models/MetricMM.cs +++ b/server/AyaNova/models/MetricMM.cs @@ -11,7 +11,7 @@ namespace AyaNova.Models { [Required] [Key] - public DateTime t { get; set; } + public DateTimeOffset t { get; set; } public long Allocated { get; set; } public long WorkingSet { get; set; } public long PrivateBytes { get; set; } @@ -25,7 +25,7 @@ namespace AyaNova.Models public MetricMM(long allocated, long workingSet, long privateBytes, int gen0, int gen1, int gen2, double cpu) { - t = System.DateTime.UtcNow; + t = System.DateTimeOffset.UtcNow; Allocated = allocated; WorkingSet = workingSet; PrivateBytes = privateBytes;