admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Browse Source
This commit is contained in:
John Cardinal
2020-05-27 21:25:07 +00:00
parent 811b7c88a2
commit 524147ee2c
4 changed files with 71 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.vscode/launch.json vendored
View File

@@ -40,8 +40,8 @@
"env": {
"ASPNETCORE_ENVIRONMENT": "Development",
"AYANOVA_JWT_SECRET": "UNLICENSED5G*QQJ8#bQ7$Xr_@sXfHq4",
//"AYANOVA_LOG_LEVEL": "Info",
"AYANOVA_LOG_LEVEL": "Debug",
"AYANOVA_LOG_LEVEL": "Info",
//"AYANOVA_LOG_LEVEL": "Debug",
"AYANOVA_DEFAULT_TRANSLATION": "en",
//TRANSLATION MUST BE en for Integration TESTING
//"AYANOVA_PERMANENTLY_ERASE_DATABASE": "true",
@@ -50,7 +50,7 @@
"AYANOVA_FOLDER_USER_FILES": "c:\\temp\\RavenTestData\\userfiles",
"AYANOVA_FOLDER_BACKUP_FILES": "c:\\temp\\RavenTestData\\backupfiles",
"AYANOVA_METRICS_USE_INFLUXDB": "false",
"AYANOVA_SERVER_TEST_MODE":"false",
"AYANOVA_SERVER_TEST_MODE":"true",
"AYANOVA_SERVER_TEST_MODE_SEEDLEVEL":"small",
"AYANOVA_SERVER_TEST_MODE_TZ_OFFSET":"-7",
"AYANOVA_BACKUP_PG_DUMP_PATH":"C:\\data\\code\\PostgreSQLPortable_12.0\\App\\PgSQL\\bin\\"

47
devdocs/todo.txt
View File

@@ -45,10 +45,57 @@ OK, after some research I am doing the correct thing, it should be:
Schema: "timestamp" Model: DateTime
The only issue may be in the interpretation of the date time, it's sent to postgres as is so if the server converts it as local from the parameter in the route then that's waht's sent
which is what I'm seeing in metric route so it's a translation of the datetime incoming to route parameter issue only and only when it hits the db for storage or query
2020-05-27 12:51:51.1443|INFO|Microsoft.EntityFrameworkCore.Database.Command|Executed DbCommand (4ms) [Parameters=[@__ToUniversalTime_0='2020-05-27T13:51:49' (DbType = DateTime),
@__ToUniversalTime_1='2020-05-27T19:51:50' (DbType = DateTime)], CommandType='Text', CommandTimeout='30']
SELECT a.t, a.allocated, a.cpu, a.gen0, a.gen1, a.gen2, a.privatebytes, a.workingset
FROM ametricmm AS a
WHERE (a.t >= @__ToUniversalTime_0) AND (a.t <= @__ToUniversalTime_1)
ORDER BY a.t
Ok, so the WHERE bit absolutely needs the to universal time to fixup the parameter that came in wrong
Storage?? I best check that shit use eventlog...
Yes event log is properly inserting with utc time:
2020-05-27 12:51:57.3213|INFO|Microsoft.EntityFrameworkCore.Database.Command|Executed DbCommand (10ms) [Parameters=[@p0='2', @p1='0', @p2='12', @p3='2020-05-27T19:51:56' (DbType = DateTime), @p4=NULL, @p5='1'], CommandType='Text', CommandTimeout='30']
INSERT INTO aevent (ayevent, ayid, ayatype, created, textra, userid)
VALUES (@p0, @p1, @p2, @p3, @p4, @p5)
RETURNING id, xmin;
Ok, that's internally, but waht about a record sent in from the client?
### Try Widget
Posted as: {"id":0,"concurrency":0,"name":"TEST DATE FUCKERY","serial":0,"dollarAmount":123,"active":null,"userType":2,"startDate":"2020-05-27T19:20:00.000Z","endDate":"2020-05-27T20:00:00.000Z","notes":"3333","count":"333","wiki":null,"customFields":"{\"c2\":\"3333\"}","tags":[],"userId":null}
12:20 to 1:00 pm was selected in client
Fetched back as:
{"data":{"id":2751,"concurrency":11535962,"name":"TEST DATE FUCKERY","serial":2751,"dollarAmount":123.0000,"active":null,"userType":2,"startDate":"2020-05-27T19:20:00Z","endDate":"2020-05-27T20:00:00Z","notes":"3333","count":333,"wiki":null,"customFields":"{\"c2\":\"3333\"}","tags":[],"userId":null}}
so identically which is good
Stored in DB AS:
Start date: 2020-05-27 19:20:00 End date: 2020-05-27 20:00:00
Ok, so a widget is utc at all aspects and is good!
## HOLUP
Ok, though I had it all worked out but then I'm seeing incoming back to client the dates from metrics of this:
2020-05-28T02:57:58.968205Z for 2020-05-27 12:57:58 Which is absolutely wrong, probably 7 hours ahead, not behind
I must be reconverting somewhere
Widget tests ok end to end, it's just in that metric controller fuckery
TODO: Why is this adding UTC bit to this query check the source code maybe something is up
Schema settings mean that pg will ignore any tz specification so in theory it's ignored, but...
SELECT a.gid, a.xmin, a.created, a.exclusive, a.jobinfo, a.jobstatus, a.jobtype, a.name, a.objectid, a.objecttype, a.startafter, a.subtype
FROM aopsjob AS a
WHERE ((a.startafter < NOW() AT TIME ZONE 'UTC') AND (a.exclusive = @__exclusiveOnly_0)) AND (a.jobstatus = 1)
ORDER BY a.created
2020-05-27 12:51:57.2585|INFO|Microsoft.EntityFrameworkCore.Database.Command|Executed DbCommand (10ms) [Parameters=[@__exclusiveOnly_0='False'], CommandType='Text', CommandTimeout='30']
SELECT a.gid, a.xmin, a.created, a.exclusive, a.jobinfo, a.jobstatus, a.jobtype, a.name, a.objectid, a.objecttype, a.startafter, a.subtype
FROM aopsjob AS a
WHERE ((a.startafter < NOW() AT TIME ZONE 'UTC') AND (a.exclusive = @__exclusiveOnly_0)) AND (a.jobstatus = 1)
ORDER BY a.created
todo: Are serial numbers working properly, I'm seeing gaps in widget but it could be because of unit testing
todo: leave running check it does overnight backup properly
todo: add backup turn off setting

39
server/AyaNova/Controllers/ServerMetricsController.cs
View File

@@ -66,9 +66,9 @@ namespace AyaNova.Api.Controllers
{
return StatusCode(403, new ApiNotAuthorizedResponse());
}
//use specified values or just return all
maxRecords ??= MAX_RECORDS_BEFORE_DOWNSAMPLING;
@@ -77,7 +77,7 @@ namespace AyaNova.Api.Controllers
//touniversal is because the parameters are converted to local time here, but then sent to the query as local time as well and not universal time which is what it should be
MinuteMetrics = await ct.MetricMM.AsNoTracking().Where(z => z.t >= tsStart.ToUniversalTime() && z.t <= tsEnd.ToUniversalTime()).OrderBy(z => z.t).ToListAsync();
//var v=await ct.MetricMM.AsNoTracking().Where(z => z.t >= tsStart.ToUniversalTime() && z.t <= tsEnd.ToUniversalTime()).OrderByDescending(z => z.t).ToListAsync();
//var v=await ct.MetricMM.AsNoTracking().Where(z => z.t >= tsStart.ToUniversalTime() && z.t <= tsEnd.ToUniversalTime()).OrderByDescending(z => z.t).ToListAsync();
//Log
await EventLogProcessor.LogEventToDatabaseAsync(new Event(UserIdFromContext.Id(HttpContext.Items), 0, AyaType.Metrics, AyaEvent.Retrieved), ct);
@@ -88,25 +88,25 @@ namespace AyaNova.Api.Controllers
{
//yes, so need to return individual labels and downsampled data as they wont' sync anymore
var dsCPU = MinuteMetrics.Select(z => new Tuple<double, double>(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.CPU)).ToList();
var dsCPU = MinuteMetrics.Select(z => new Tuple<double, double>(z.t.ToOADate(), z.CPU)).ToList();
dsCPU = Util.DataUtil.LargestTriangleThreeBuckets(dsCPU, (int)maxRecords) as List<Tuple<double, double>>;
var dsAllocated = MinuteMetrics.Select(z => new Tuple<double, double>(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.Allocated)).ToList();
var dsAllocated = MinuteMetrics.Select(z => new Tuple<double, double>(z.t.ToOADate(), z.Allocated)).ToList();
dsAllocated = Util.DataUtil.LargestTriangleThreeBuckets(dsAllocated, (int)maxRecords) as List<Tuple<double, double>>;
var dsWorkingSet = MinuteMetrics.Select(z => new Tuple<double, double>(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.WorkingSet)).ToList();
var dsWorkingSet = MinuteMetrics.Select(z => new Tuple<double, double>(z.t.ToOADate(), z.WorkingSet)).ToList();
dsWorkingSet = Util.DataUtil.LargestTriangleThreeBuckets(dsWorkingSet, (int)maxRecords) as List<Tuple<double, double>>;
var dsPrivateBytes = MinuteMetrics.Select(z => new Tuple<double, double>(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.PrivateBytes)).ToList();
var dsPrivateBytes = MinuteMetrics.Select(z => new Tuple<double, double>(z.t.ToOADate(), z.PrivateBytes)).ToList();
dsPrivateBytes = Util.DataUtil.LargestTriangleThreeBuckets(dsPrivateBytes, (int)maxRecords) as List<Tuple<double, double>>;
var dsGen0 = MinuteMetrics.Select(z => new Tuple<double, double>(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.Gen0)).ToList();
var dsGen0 = MinuteMetrics.Select(z => new Tuple<double, double>(z.t.ToOADate(), z.Gen0)).ToList();
dsGen0 = Util.DataUtil.LargestTriangleThreeBuckets(dsGen0, (int)maxRecords) as List<Tuple<double, double>>;
var dsGen1 = MinuteMetrics.Select(z => new Tuple<double, double>(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.Gen1)).ToList();
var dsGen1 = MinuteMetrics.Select(z => new Tuple<double, double>(z.t.ToOADate(), z.Gen1)).ToList();
dsGen1 = Util.DataUtil.LargestTriangleThreeBuckets(dsGen1, (int)maxRecords) as List<Tuple<double, double>>;
var dsGen2 = MinuteMetrics.Select(z => new Tuple<double, double>(new DateTimeOffset(z.t).ToUnixTimeSeconds(), z.Gen2)).ToList();
var dsGen2 = MinuteMetrics.Select(z => new Tuple<double, double>(z.t.ToOADate(), z.Gen2)).ToList();
dsGen2 = Util.DataUtil.LargestTriangleThreeBuckets(dsGen2, (int)maxRecords) as List<Tuple<double, double>>;
@@ -118,19 +118,19 @@ namespace AyaNova.Api.Controllers
DownSampled = true,
MetricMM = new
{
cpuLabels = dsCPU.Select(z => DateTimeOffset.FromUnixTimeSeconds((long)z.Item1)).ToArray(),
cpuLabels = dsCPU.Select(z => DateTime.FromOADate(z.Item1)).ToArray(),
cpu = dsCPU.Select(z => z.Item2).ToArray(),
get0Labels = dsGen0.Select(z => DateTimeOffset.FromUnixTimeSeconds((long)z.Item1)).ToArray(),
get0Labels = dsGen0.Select(z => DateTime.FromOADate(z.Item1)).ToArray(),
gen0 = dsGen0.Select(z => (int)z.Item2).ToArray(),
get1Labels = dsGen1.Select(z => DateTimeOffset.FromUnixTimeSeconds((long)z.Item1)).ToArray(),
get1Labels = dsGen1.Select(z => DateTime.FromOADate(z.Item1)).ToArray(),
gen1 = dsGen1.Select(z => (int)z.Item2).ToArray(),
get2Labels = dsGen2.Select(z => DateTimeOffset.FromUnixTimeSeconds((long)z.Item1)).ToArray(),
get2Labels = dsGen2.Select(z => DateTime.FromOADate(z.Item1)).ToArray(),
gen2 = dsGen2.Select(z => (int)z.Item2).ToArray(),
allocatedLabels = dsAllocated.Select(z => DateTimeOffset.FromUnixTimeSeconds((long)z.Item1)).ToArray(),
allocatedLabels = dsAllocated.Select(z => DateTime.FromOADate(z.Item1)).ToArray(),
allocated = dsAllocated.Select(z => (long)z.Item2).ToArray(),
workingSetLabels = dsWorkingSet.Select(z => DateTimeOffset.FromUnixTimeSeconds((long)z.Item1)).ToArray(),
workingSetLabels = dsWorkingSet.Select(z => DateTime.FromOADate(z.Item1)).ToArray(),
workingSet = dsWorkingSet.Select(z => (long)z.Item2).ToArray(),
privateBytesLabels = dsPrivateBytes.Select(z => DateTimeOffset.FromUnixTimeSeconds((long)z.Item1)).ToArray(),
privateBytesLabels = dsPrivateBytes.Select(z => DateTime.FromOADate(z.Item1)).ToArray(),
privateBytes = dsPrivateBytes.Select(z => (long)z.Item2).ToArray()
}
@@ -147,7 +147,8 @@ namespace AyaNova.Api.Controllers
DownSampled = false,
MetricMM = new
{
labels = MinuteMetrics.Select(z => z.t.ToUniversalTime()).ToArray(),//db is in UTC, ef core retrieves as if it was local
//labels = MinuteMetrics.Select(z => z.t.ToUniversalTime()).ToArray(),//db is in UTC, ef core retrieves as if it was local
labels = MinuteMetrics.Select(z => z.t).ToArray(),//db is in UTC, ef core retrieves as if it was local
cpu = MinuteMetrics.Select(z => z.CPU).ToArray(),
gen0 = MinuteMetrics.Select(z => z.Gen0).ToArray(),
gen1 = MinuteMetrics.Select(z => z.Gen1).ToArray(),

2
server/AyaNova/generator/CoreJobMetricsSnapshot.cs
View File

@@ -24,7 +24,7 @@ namespace AyaNova.Biz
private static double _cpu = 0;
#if(DEBUG)
private static TimeSpan tsOneMinute = new TimeSpan(0, 0,10);
private static TimeSpan tsOneMinute = new TimeSpan(0, 0,5);
#else
private static TimeSpan tsOneMinute = new TimeSpan(0, 1, 0);
#endif