admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Browse Source
This commit is contained in:
John Cardinal
2020-05-27 19:41:26 +00:00
parent 5449632edc
commit f9f29eedad
6 changed files with 79 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.vscode/launch.json vendored
View File

@@ -40,8 +40,8 @@
"env": { "env": {
"ASPNETCORE_ENVIRONMENT": "Development", "ASPNETCORE_ENVIRONMENT": "Development",
"AYANOVA_JWT_SECRET": "UNLICENSED5G*QQJ8#bQ7$Xr_@sXfHq4", "AYANOVA_JWT_SECRET": "UNLICENSED5G*QQJ8#bQ7$Xr_@sXfHq4",
"AYANOVA_LOG_LEVEL": "Info", //"AYANOVA_LOG_LEVEL": "Info",
//"AYANOVA_LOG_LEVEL": "Debug", "AYANOVA_LOG_LEVEL": "Debug",
"AYANOVA_DEFAULT_TRANSLATION": "en", "AYANOVA_DEFAULT_TRANSLATION": "en",
//TRANSLATION MUST BE en for Integration TESTING //TRANSLATION MUST BE en for Integration TESTING
//"AYANOVA_PERMANENTLY_ERASE_DATABASE": "true", //"AYANOVA_PERMANENTLY_ERASE_DATABASE": "true",

42
devdocs/todo.txt
View File

@@ -1,6 +1,48 @@
{"login": "manager","password": "l3tm3in"} {"login": "manager","password": "l3tm3in"}
todo: MAYBE a datetime kind of parameter is interpreted as LOCAL by the server on a route
so what the fuck
sent from client http://localhost:7575/api/v8.0/server-metric?maxRecords=400&tsStart=2020-05-27T13:07:39.068Z&tsEnd=2020-05-27T19:07:40.068Z
At server route: tsStart: {2020-05-27 6:07:39 AM} tsEnd: {2020-05-27 12:07:40 PM} Kind=local for both
so actually that's all good so far, how about the query itself?
2020-05-27 12:13:38.6439|DEBUG|Microsoft.EntityFrameworkCore.Database.Command|Executing DbCommand [Parameters=[@__tsStart_0='2020-05-27T06:13:32' (DbType = DateTime), @__tsEnd_1='2020-05-27T12:13:33' (DbType = DateTime)], CommandType='Text', CommandTimeout='30']
SELECT a.t, a.allocated, a.cpu, a.gen0, a.gen1, a.gen2, a.privatebytes, a.workingset
FROM ametricmm AS a
WHERE (a.t >= @__tsStart_0) AND (a.t <= @__tsEnd_1)
ORDER BY a.t DESC
2020-05-27 12:13:38.8848|INFO|Microsoft.EntityFrameworkCore.Database.Command|Executed DbCommand (241ms) [Parameters=[@__tsStart_0='2020-05-27T06:13:32' (DbType = DateTime), @__tsEnd_1='2020-05-27T12:13:33' (DbType = DateTime)], CommandType='Text', CommandTimeout='30']
SELECT a.t, a.allocated, a.cpu, a.gen0, a.gen1, a.gen2, a.privatebytes, a.workingset
FROM ametricmm AS a
WHERE (a.t >= @__tsStart_0) AND (a.t <= @__tsEnd_1)
ORDER BY a.t DESC
Well, shit. The query is sending local time to the db server, so any query that sends to teh server needs teh values converted to utc first
This is malarky pure and simple!
:)
If change parameters to DateTimeOffset from DateTime and rerun exact steps as above:
2020-05-27 12:21:49.8196|INFO|Microsoft.EntityFrameworkCore.Database.Command|Executed DbCommand (21ms) [Parameters=[@__tsStart_0='2020-05-27T13:21:48.4840000+00:00', @__tsEnd_1='2020-05-27T19:21:49.4840000+00:00'], CommandType='Text', CommandTimeout='30']
SELECT a.t, a.allocated, a.cpu, a.gen0, a.gen1, a.gen2, a.privatebytes, a.workingset
FROM ametricmm AS a
WHERE (CAST(a.t AS timestamp with time zone) >= @__tsStart_0) AND (CAST(a.t AS timestamp with time zone) <= @__tsEnd_1)
ORDER BY a.t
* Now it properly is sending the value to the db in UTC as it stays IN utc from input to query
One thing though is it's casting the query to a timestamp with timezone which is weird but maybe because the column is defined as timestamp witout timezone
going to try
changing the model to datetimeoffset
change database schema for mmetric to with timezone
no, actually for me it's better without because it will never try to interpret or change it, it will just return and accept it as is and
that's what I want if I'm only dealing in UTC
rerun again and see if it's casting still
todo: leave running check it does overnight backup properly todo: leave running check it does overnight backup properly
todo: add backup turn off setting todo: add backup turn off setting

39
server/AyaNova/Controllers/ServerMetricsController.cs
View File

@@ -57,7 +57,7 @@ namespace AyaNova.Api.Controllers
/// <param name="maxRecords">Optional maximum records to return. If there are more records for the time period selected than this value the result will be downsampled. There is a 400 record maximum fixed default</param> /// <param name="maxRecords">Optional maximum records to return. If there are more records for the time period selected than this value the result will be downsampled. There is a 400 record maximum fixed default</param>
/// <returns>Snapshot of metrics</returns> /// <returns>Snapshot of metrics</returns>
[HttpGet] [HttpGet]
public async Task<IActionResult> GetMetrics([FromQuery] DateTime tsStart, [FromQuery] DateTime tsEnd, [FromQuery] int? maxRecords) public async Task<IActionResult> GetMetrics([FromQuery] DateTimeOffset tsStart, [FromQuery] DateTimeOffset tsEnd, [FromQuery] int? maxRecords)
{ {
if (serverState.IsClosed) if (serverState.IsClosed)
return StatusCode(503, new ApiErrorResponse(serverState.ApiErrorCode, null, serverState.Reason)); return StatusCode(503, new ApiErrorResponse(serverState.ApiErrorCode, null, serverState.Reason));
@@ -72,6 +72,7 @@ namespace AyaNova.Api.Controllers
List<MetricMM> MinuteMetrics = new List<MetricMM>(); List<MetricMM> MinuteMetrics = new List<MetricMM>();
MinuteMetrics = await ct.MetricMM.AsNoTracking().Where(z => z.t >= tsStart && z.t <= tsEnd).OrderBy(z => z.t).ToListAsync(); MinuteMetrics = await ct.MetricMM.AsNoTracking().Where(z => z.t >= tsStart && z.t <= tsEnd).OrderBy(z => z.t).ToListAsync();
//var v=await ct.MetricMM.AsNoTracking().Where(z => z.t >= tsStart.ToUniversalTime() && z.t <= tsEnd.ToUniversalTime()).OrderByDescending(z => z.t).ToListAsync();
//Log //Log
await EventLogProcessor.LogEventToDatabaseAsync(new Event(UserIdFromContext.Id(HttpContext.Items), 0, AyaType.Metrics, AyaEvent.Retrieved), ct); await EventLogProcessor.LogEventToDatabaseAsync(new Event(UserIdFromContext.Id(HttpContext.Items), 0, AyaType.Metrics, AyaEvent.Retrieved), ct);
@@ -82,25 +83,47 @@ namespace AyaNova.Api.Controllers
{ {
//yes, so need to return individual labels and downsampled data as they wont' sync anymore //yes, so need to return individual labels and downsampled data as they wont' sync anymore
var dsCPU = MinuteMetrics.Select(z => new Tuple<double, double>(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.CPU)).ToList(); // var dsCPU = MinuteMetrics.Select(z => new Tuple<double, double>(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.CPU)).ToList();
// dsCPU = Util.DataUtil.LargestTriangleThreeBuckets(dsCPU, (int)maxRecords) as List<Tuple<double, double>>;
// var dsAllocated = MinuteMetrics.Select(z => new Tuple<double, double>(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.Allocated)).ToList();
// dsAllocated = Util.DataUtil.LargestTriangleThreeBuckets(dsAllocated, (int)maxRecords) as List<Tuple<double, double>>;
// var dsWorkingSet = MinuteMetrics.Select(z => new Tuple<double, double>(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.WorkingSet)).ToList();
// dsWorkingSet = Util.DataUtil.LargestTriangleThreeBuckets(dsWorkingSet, (int)maxRecords) as List<Tuple<double, double>>;
// var dsPrivateBytes = MinuteMetrics.Select(z => new Tuple<double, double>(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.PrivateBytes)).ToList();
// dsPrivateBytes = Util.DataUtil.LargestTriangleThreeBuckets(dsPrivateBytes, (int)maxRecords) as List<Tuple<double, double>>;
// var dsGen0 = MinuteMetrics.Select(z => new Tuple<double, double>(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.Gen0)).ToList();
// dsGen0 = Util.DataUtil.LargestTriangleThreeBuckets(dsGen0, (int)maxRecords) as List<Tuple<double, double>>;
// var dsGen1 = MinuteMetrics.Select(z => new Tuple<double, double>(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.Gen1)).ToList();
// dsGen1 = Util.DataUtil.LargestTriangleThreeBuckets(dsGen1, (int)maxRecords) as List<Tuple<double, double>>;
// var dsGen2 = MinuteMetrics.Select(z => new Tuple<double, double>(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.Gen2)).ToList();
// dsGen2 = Util.DataUtil.LargestTriangleThreeBuckets(dsGen2, (int)maxRecords) as List<Tuple<double, double>>;
var dsCPU = MinuteMetrics.Select(z => new Tuple<double, double>(z.t.ToUnixTimeSeconds(), z.CPU)).ToList();
dsCPU = Util.DataUtil.LargestTriangleThreeBuckets(dsCPU, (int)maxRecords) as List<Tuple<double, double>>; dsCPU = Util.DataUtil.LargestTriangleThreeBuckets(dsCPU, (int)maxRecords) as List<Tuple<double, double>>;
var dsAllocated = MinuteMetrics.Select(z => new Tuple<double, double>(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.Allocated)).ToList(); var dsAllocated = MinuteMetrics.Select(z => new Tuple<double, double>(z.t.ToUnixTimeSeconds(), z.Allocated)).ToList();
dsAllocated = Util.DataUtil.LargestTriangleThreeBuckets(dsAllocated, (int)maxRecords) as List<Tuple<double, double>>; dsAllocated = Util.DataUtil.LargestTriangleThreeBuckets(dsAllocated, (int)maxRecords) as List<Tuple<double, double>>;
var dsWorkingSet = MinuteMetrics.Select(z => new Tuple<double, double>(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.WorkingSet)).ToList(); var dsWorkingSet = MinuteMetrics.Select(z => new Tuple<double, double>(z.t.ToUnixTimeSeconds(), z.WorkingSet)).ToList();
dsWorkingSet = Util.DataUtil.LargestTriangleThreeBuckets(dsWorkingSet, (int)maxRecords) as List<Tuple<double, double>>; dsWorkingSet = Util.DataUtil.LargestTriangleThreeBuckets(dsWorkingSet, (int)maxRecords) as List<Tuple<double, double>>;
var dsPrivateBytes = MinuteMetrics.Select(z => new Tuple<double, double>(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.PrivateBytes)).ToList(); var dsPrivateBytes = MinuteMetrics.Select(z => new Tuple<double, double>(z.t.ToUnixTimeSeconds(), z.PrivateBytes)).ToList();
dsPrivateBytes = Util.DataUtil.LargestTriangleThreeBuckets(dsPrivateBytes, (int)maxRecords) as List<Tuple<double, double>>; dsPrivateBytes = Util.DataUtil.LargestTriangleThreeBuckets(dsPrivateBytes, (int)maxRecords) as List<Tuple<double, double>>;
var dsGen0 = MinuteMetrics.Select(z => new Tuple<double, double>(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.Gen0)).ToList(); var dsGen0 = MinuteMetrics.Select(z => new Tuple<double, double>(z.t.ToUnixTimeSeconds(), z.Gen0)).ToList();
dsGen0 = Util.DataUtil.LargestTriangleThreeBuckets(dsGen0, (int)maxRecords) as List<Tuple<double, double>>; dsGen0 = Util.DataUtil.LargestTriangleThreeBuckets(dsGen0, (int)maxRecords) as List<Tuple<double, double>>;
var dsGen1 = MinuteMetrics.Select(z => new Tuple<double, double>(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.Gen1)).ToList(); var dsGen1 = MinuteMetrics.Select(z => new Tuple<double, double>(z.t.ToUnixTimeSeconds(), z.Gen1)).ToList();
dsGen1 = Util.DataUtil.LargestTriangleThreeBuckets(dsGen1, (int)maxRecords) as List<Tuple<double, double>>; dsGen1 = Util.DataUtil.LargestTriangleThreeBuckets(dsGen1, (int)maxRecords) as List<Tuple<double, double>>;
var dsGen2 = MinuteMetrics.Select(z => new Tuple<double, double>(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.Gen2)).ToList(); var dsGen2 = MinuteMetrics.Select(z => new Tuple<double, double>(z.t.ToUnixTimeSeconds(), z.Gen2)).ToList();
dsGen2 = Util.DataUtil.LargestTriangleThreeBuckets(dsGen2, (int)maxRecords) as List<Tuple<double, double>>; dsGen2 = Util.DataUtil.LargestTriangleThreeBuckets(dsGen2, (int)maxRecords) as List<Tuple<double, double>>;

2
server/AyaNova/Startup.cs
View File

@@ -134,7 +134,7 @@ namespace AyaNova
bool LOG_SENSITIVE_DATA = false; bool LOG_SENSITIVE_DATA = false;
#if (DEBUG) #if (DEBUG)
LOG_SENSITIVE_DATA = false;//############################################################################ LOG_SENSITIVE_DATA = true;//############################################################################
#endif #endif

2
server/AyaNova/generator/CoreJobMetricsSnapshot.cs
View File

@@ -24,7 +24,7 @@ namespace AyaNova.Biz
private static double _cpu = 0; private static double _cpu = 0;
#if(DEBUG) #if(DEBUG)
private static TimeSpan tsOneMinute = new TimeSpan(0, 1,0); private static TimeSpan tsOneMinute = new TimeSpan(0, 0,10);
#else #else
private static TimeSpan tsOneMinute = new TimeSpan(0, 1, 0); private static TimeSpan tsOneMinute = new TimeSpan(0, 1, 0);
#endif #endif

4
server/AyaNova/models/MetricMM.cs
View File

@@ -11,7 +11,7 @@ namespace AyaNova.Models
{ {
[Required] [Required]
[Key] [Key]
public DateTime t { get; set; } public DateTimeOffset t { get; set; }
public long Allocated { get; set; } public long Allocated { get; set; }
public long WorkingSet { get; set; } public long WorkingSet { get; set; }
public long PrivateBytes { get; set; } public long PrivateBytes { get; set; }
@@ -25,7 +25,7 @@ namespace AyaNova.Models
public MetricMM(long allocated, long workingSet, long privateBytes, int gen0, int gen1, int gen2, double cpu) public MetricMM(long allocated, long workingSet, long privateBytes, int gen0, int gen1, int gen2, double cpu)
{ {
t = System.DateTime.UtcNow; t = System.DateTimeOffset.UtcNow;
Allocated = allocated; Allocated = allocated;
WorkingSet = workingSet; WorkingSet = workingSet;
PrivateBytes = privateBytes; PrivateBytes = privateBytes;