206 lines
34 KiB
C#
206 lines
34 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Threading.Tasks;
|
|
using System.Diagnostics;
|
|
using Microsoft.Extensions.Logging;
|
|
using AyaNova.Util;
|
|
using AyaNova.Models;
|
|
|
|
|
|
|
|
namespace AyaNova.Biz
|
|
{
|
|
/// <summary>
|
|
/// called by Generator to gather server metrics and insert in db
|
|
/// </summary>
|
|
internal static class CoreJobMetricsSnapshot
|
|
{
|
|
private static ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("CoreJobMetricsSnapshot");
|
|
|
|
#if (DEBUG)
|
|
private static TimeSpan DO_EVERY_INTERVAL = new TimeSpan(0, 1, 0);//DEBUG do a check every 60 seconds
|
|
#else
|
|
private static TimeSpan DO_EVERY_INTERVAL = new TimeSpan(0, 15, 0);//RELEASE do a check every 15 minutes
|
|
#endif
|
|
|
|
private static DateTime lastServerCheckDone = DateTime.MinValue;
|
|
private static DateTime lastRecordCountCheck = DateTime.MinValue;
|
|
private static DateTime lastFileCountCheck = DateTime.MinValue;
|
|
|
|
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
// DoAsync
|
|
//
|
|
public static async Task DoJobAsync(AyContext ct)
|
|
{
|
|
//Gather stats, output to database but only every minute or more
|
|
/*
|
|
//TODO: figure out teh best format to store it in based on what I need at the client end
|
|
// retention setting defaults to 1 year?
|
|
Generate sample data, see how large it would be under various scenarios
|
|
i.e. if I gather every minute, how much can I practically store?
|
|
Shortest time frame that is shown for DO is 1 minute intervals (6 hour / 360 entries)
|
|
RESULT:
|
|
525600 entries (every minute for 1 year) results in 22mb of space used
|
|
query: insert into ametriccpu (t,v) select CURRENT_TIMESTAMP, 58.43239007949476 from generate_series(1, 525600) s(i)
|
|
|
|
Timestamp, decimal/float value (need to determine this number type)
|
|
Number types do has this:
|
|
CPU avg: 0.33006058073955513
|
|
transmit bytes: 102.44723488288768
|
|
receive bytes: 46.078341513002755
|
|
disk usage avg: (percentage) 30.23570573933185
|
|
disk sectors written: 0.000136518543824419
|
|
memory avg: 65.8400891412282
|
|
load: 0.01
|
|
Make a chart at client with test data from digital ocean to play with
|
|
try to replicate their stuff to learn how to best do it
|
|
Downsampling, should I convert old data to downsampled so I can keep more of it or is this presentation only
|
|
depends on storage space I guess
|
|
seperate tables per metric?
|
|
Seems likely since there would be less data to move around, but if I'm fetching all anyway??
|
|
hmm... something to consider / experiment with
|
|
|
|
Some D.O. charts have max value at top of left axis (y?) that change with the range and values and some have 100% or 1.0 and never change that axis
|
|
|
|
|
|
|
|
|
|
|
|
|
|
Digital ocean has a seperate query for each type of info
|
|
Implying maybe seperate storage for each type which makes sense I guess
|
|
though their data comes from all over the place
|
|
|
|
For a 30 day period their data came across like this:
|
|
109 minutes apart
|
|
CPU average: 396 entries 1.8 times per hour or every 109 minutes
|
|
Disk usage: 400 entries every 108 minutes (peculiar)
|
|
|
|
For 6 hour period get 360 entries?!
|
|
1 minute apart
|
|
they are all today, so it's always returning the same number of entries, regardless of time period
|
|
they are one minute apart
|
|
so it must be sampling somehow for the longer periods at least
|
|
|
|
for 24 hours get 401 entries
|
|
3.6 minutes apart
|
|
|
|
7 days get 400 entries
|
|
25 minutes apart
|
|
|
|
|
|
OK, not a trivial thing to graph it requires "downsampling" or "decimation"
|
|
Here is a link to a c# method that will accomplish this:
|
|
https://github.com/sveinn-steinarsson/flot-downsample/
|
|
c#
|
|
https://gist.github.com/DanielWJudge/63300889f27c7f50eeb7
|
|
|
|
|
|
CREATE flat_table(
|
|
trip_id integer,
|
|
tstamp timestamptz,
|
|
speed float,
|
|
distance float,
|
|
temperature float,
|
|
,...);
|
|
|
|
actual data:
|
|
|
|
{"status":"success","data":{"resultType":"matrix","result":[{"metric":{},"values":[[1587768134,"51.542056626728986"],[1587774614,"52.020442635021006"],[1587781094,"52.37834624122467"],[1587787574,"52.76066131451813"],[1587794054,"53.076435453324876"],[1587800534,"44.07096649723009"],[1587807014,"44.70212104150343"],[1587813494,"45.22263651719236"],[1587819974,"45.74512065958209"],[1587826454,"53.23471625606842"],[1587832934,"53.66861039692258"],[1587839414,"53.965879068741906"],[1587845894,"54.20763133959894"],[1587852374,"52.61891731206123"],[1587858854,"53.1362829210289"],[1587865334,"53.43512652620886"],[1587871814,"53.82925359970706"],[1587878294,"54.01745813630261"],[1587884774,"46.972387480854714"],[1587891254,"47.647246425885605"],[1587897734,"47.91813496391434"],[1587904214,"48.1783927017588"],[1587910694,"56.29796164249801"],[1587917174,"55.89674736887695"],[1587923654,"56.29402430909642"],[1587930134,"56.56136924706373"],[1587936614,"56.62791018155045"],[1587943094,"56.18732257391359"],[1587949574,"56.39285137747609"],[1587956054,"55.85540536816036"],[1587962534,"56.63735978171424"],[1587969014,"52.26101370585757"],[1587975494,"49.91672539855657"],[1587981974,"50.097842735029275"],[1587988454,"51.35818315687517"],[1587994934,"57.60869993188413"],[1588001414,"57.74453793423866"],[1588007894,"58.00203953870202"],[1588014374,"58.25639127644412"],[1588020854,"58.43239007949476"],[1588027334,"58.66863008358959"],[1588033814,"59.46042783064741"],[1588040294,"59.706117434906034"],[1588046774,"59.742340902200574"],[1588053254,"59.74745943562263"],[1588059734,"51.33455915646569"],[1588066214,"51.8243634316223"],[1588072694,"52.80594064863631"],[1588079174,"52.968552518121584"],[1588085654,"60.73258025269806"],[1588092134,"60.15930450942795"],[1588098614,"60.27978691151631"],[1588105094,"60.449092247784264"],[1588111574,"60.51090838218908"],[1588118054,"60.247894510963505"],[1588124534,"56.15936750676237"],[1588131014,"56.37828324389025"],[1588137494,"55.933758302851814"],[1588143974,"51.0483150181708"],[1588150454,"51.368813957059444"],[1588156934,"51.647577161891334"],[1588163414,"52.327554640344275"],[1588169894,"58.49971848066179"],[1588176374,"58.60130168242256"],[1588182854,"59.11472995798866"],[1588189334,"59.69273050134066"],[1588195814,"59.70218010150445"],[1588202294,"59.61083396658778"],[1588208774,"60.3727079797936"],[1588215254,"60.589261316880524"],[1588221734,"60.67824505175625"],[1588228214,"57.0074691214628"],[1588234694,"52.60553037849586"],[1588241174,"52.750030514333865"],[1588247654,"52.844920249311954"],[1588254134,"59.189539292618676"],[1588260614,"59.525393831773485"],[1588267094,"59.73249756869663"],[1588273574,"59.887234771378736"],[1588280054,"60.09000744156013"],[1588286534,"60.28529917827852"],[1588293014,"55.68767496525303"],[1588299494,"56.61531071466539"],[1588305974,"56.97754538761078"],[1588312454,"57.08070352273219"],[1588318934,"52.54292677741073"],[1588325414,"52.75042424767402"],[1588331894,"52.92484811736403"],[1588338374,"53.02997491918623"],[1588344854,"65.66724020489883"],[1588351334,"65.79835340717146"],[1588357814,"65.3577657995346"],[1588364294,"59.78683276963843"],[1588370774,"60.39200091346135"],[1588377254,"60.567212249831684"],[1588383734,"61.894487339504444"],[1588390214,"62.03308147524008"],[1588396694,"61.97559640757701"],[1588403174,"53.87610786718587"],[1588409654,"54.12219120478464"],[1588416134,"54.370243209084215"],[1588422614,"54.50214387803716"],[1588429094,"62.339406013883035"],[1588435574,"59.50255729804432"],[1588442054,"59.81360663676918"],[1588448534,"59.98488063973794"],[1588455014,"60.113631441969616"],[1588461494,"60.255375444426505"],[1588467974,"60.48649691509927"],[1588474454,"60.56524358313089"],[1588480934,"60.67470145169482"],[1588487414,"57.417345528567324"],[1588493894,"54.637981880391685"],[1588500374,"54.83287988376991"],[1588506854,"55.1990518901169"],[1588513334,"62.47721268293836"],[1588519814,"63.006784025450926"],[1588526294,"63.28475976360251"],[1588532774,"63.351300698089204"],[1588539254,"63.70211710417003"],[1588545734,"63.82417443961902"],[1588552214,"63.90961457443331"],[1588558694,"64.04624004346816"],[1588565174,"64.16239137881479"],[1588571654,"64.24546911358813"],[1588578134,"55.72114229916647"],[1588584614,"55.98888097047394"],[1588591094,"56.145980573196994"],[1588597574,"56.13692470637336"],[1588604054,"62.8953574901862"],[1588610534,"62.59178908492434"],[1588617014,"62.746132554266296"],[1588623494,"62.88236428996098"],[1588629974,"63.34027616456479"],[1588636454,"63.80685017265208"],[1588642934,"64.1312864449423"],[1588649414,"64.40217498297103"],[1588655894,"64.66479512085644"],[1588662374,"56.281031108871204"],[1588668854,"56.79367191775698"],[1588675334,"56.839738718555466"],[1588681814,"57.234259525393824"],[1588688294,"65.77433567342182"],[1588694774,"65.76803593997928"],[1588701254,"65.82079620756048"],[1588707734,"66.50510475275514"],[1588714214,"66.48069328566535"],[1588720694,"66.31020674937692"],[1588727174,"64.84236885726773"],[1588733654,"64.64707712054934"],[1588740134,"64.82228845691967"],[1588746614,"62.07875454269841"],[1588753094,"55.63215856429075"],[1588759574,"55.6955496320562"],[1588766054,"55.75775949980116"],[1588772534,"63.28987829702456"],[1588779014,"63.52729950113986"],[1588785494,"63.93441977486327"],[1588791974,"64.12774284488087"],[1588798454,"64.70534965489273"],[1588804934,"57.931561270813724"],[1588811414,"57.955579004563376"],[1588817894,"58.00715807212408"],[1588824374,"58.12173447411006"],[1588830854,"58.300489410541815"],[1588837334,"55.20259549017832"],[1588843814,"56.10227617243946"],[1588850294,"56.15661137338126"],[1588856774,"56.22393977454829"],[1588863254,"63.04891349284784"],[1588869734,"64.34350871528748"],[1588876214,"63.96040617531371"],[1588882694,"64.19192137932663"],[1588889174,"64.09309431094697"],[1588895654,"64.25688738045271"],[1588902134,"63.71944137113699"],[1588908614,"63.92497017469948"],[1588915094,"64.12104937809819"],[1588921574,"55.9479327030975"],[1588928054,"55.649876564597854"],[1588934534,"55.750672299678314"],[1588941014,"55.86367376830368"],[1588947494,"62.53824135066285"],[1588953974,"62.677622953078796"],[1588960454,"62.461463349332035"],[1588966934,"63.046157359466726"],[1588973414,"63.60525870249115"],[1588979894,"63.746608971607884"],[1588986374,"64.45336031719158"],[1588992854,"64.54549391878857"],[1588999334,"63.999385775989346"],[1589005814,"59.296241027801514"],[1589012294,"56.45033644513916"],[1589018774,"56.51963351300699"],[1589025254,"56.64680938187803"],[1589031734,"64.08403844412334"],[1589038214,"64.1950712460479"],[1589044694,"64.24861898030939"],[1589051174,"60.40145051362514"],[1589057654,"60.43491784753857"],[1589064134,"60.52941384917651"],[1589070614,"60.711318652329524"],[1589077094,"60.75620425310754"],[1589083574,"60.88731745538018"],[1589090054,"60.956220789907825"],[1589096534,"56.18653510723327"],[1589103014,"56.94801538709894"],[1589109494,"57.14094472377638"],[1589115974,"57.29883179317975"],[1589122454,"64.34468991530797"],[1589128934,"64.43367365018368"],[1589135414,"64.57226778591931"],[1589141894,"64.80023938987081"],[1589148374,"64.9758444595813"],[1589154854,"65.16523019619733"],[1589161334,"65.20657219691391"],[1589167814,"65.38847700006694"],[1589174294,"65.46171140133633"],[1589180774,"55.98888097047394"],[1589187254,"56.30071777587911"],[1589193734,"56.54798231349836"],[1589200214,"57.264576992586"],[1589206694,"63.6985735041086"],[1589213174,"63.1619149614732"],[1589219654,"63.45327763319014"],[1589226134,"63.68558030388339"],[1589232614,"63.87063497375768"],[1589239094,"64.05096484355006"],[1589245574,"65.18412939652491"],[1589252054,"64.95852019261436"],[1589258534,"64.82898192370236"],[1589265014,"60.18765330991932"],[1589271494,"56.716500183086005"],[1589277974,"56.7306745833317"],[1589284454,"56.94959032045956"],[1589290934,"64.22853857996134"],[1589297414,"59.61634623335"],[1589303894,"60.08370770811759"],[1589310374,"60.19316557668154"],[1589316854,"58.702884884183334"],[1589323334,"59.011571822867246"],[1589329814,"59.25962382716682"],[1589336294,"59.39821796290244"],[1589342774,"59.480508230995476"],[1589349254,"59.56279849908851"],[1589355734,"55.39316242681481"],[1589362214,"55.45379736119915"],[1589368694,"55.59829749703715"],[1589375174,"55.74122269951452"],[1589381654,"63.815512306135545"],[1589388134,"64.00332310939093"],[1589394614,"58.54893514818155"],[1589401094,"59.167096492229675"],[1589407574,"59.240724626839224"],[1589414054,"59.47617716425374"],[1589420534,"60.130168242256254"],[1589427014,"59.877391437874785"],[1589433494,"59.98094330633636"],[1589439974,"56.420018977946995"],[1589446454,"56.794065651097135"],[1589452934,"56.97557672090999"],[1589459414,"56.931084853472136"],[1589465894,"58.31702621082846"],[1589472374,"58.846203820000866"],[1589478854,"58.95408675520417"],[1589485334,"59.17221502565173"],[1589491814,"59.3714440957717"],[1589498294,"59.584060099457034"],[1589504774,"59.792345036400654"],[1589511254,"59.908102638407115"],[1589517734,"60.09512597498219"],[1589524214,"59.02732115647357"],[1589530694,"56.607829781202376"],[1589537174,"56.6995696494592"],[1589543654,"56.83501391847358"],[1589550134,"63.209556695632315"],[1589556614,"63.340669897904945"],[1589563094,"63.504069234070535"],[1589569574,"63.65683777005186"],[1589576054,"63.94820044176881"],[1589582534,"63.99505470924761"],[1589589014,"64.5423440520673"],[1589595494,"64.98371912638447"],[1589601974,"65.10695766185394"],[1589608454,"65.57195673658059"],[1589614934,"55.69594336539635"],[1589621414,"55.97313163686761"],[1589627894,"56.786190984293974"],[1589634374,"56.817295918166465"],[1589640854,"61.338142129861126"],[1589647334,"61.527921599817304"],[1589653814,"61.64486040184425"],[1589660294,"61.77833600415782"],[1589666774,"61.821646671575216"],[1589673254,"61.968902940794315"],[1589679734,"62.21931734513484"],[1589686214,"62.27365254607664"],[1589692694,"62.38822894806264"],[1589699174,"55.683343898511296"],[1589705654,"56.56688151382595"],[1589712134,"56.67437071568909"],[1589718614,"56.70547564956158"],[1589725094,"64.42422405001989"],[1589731574,"64.55297485225157"],[1589738054,"64.66558258753676"],[1589744534,"64.83921899054647"],[1589751014,"64.98883765980652"],[1589757494,"65.08608979482555"],[1589763974,"65.423519267341"],[1589770454,"65.5924308702688"],[1589776934,"65.77906047350372"],[1589783414,"63.95331897519087"],[1589789894,"56.08967670555439"],[1589796374,"56.26410057524441"],[1589802854,"56.41726284456588"],[1589809334,"63.49343843388626"],[1589815814,"63.809606306033174"],[1589822294,"61.586981600841014"],[1589828774,"61.63737946838125"],[1589835254,"61.97914000763842"],[1589841734,"62.15789494407018"],[1589848214,"62.45083254914776"],[1589854694,"62.653211485989004"],[1589861174,"62.82251682225696"],[1589867654,"62.358698947550785"],[1589874134,"55.99084963717472"],[1589880614,"56.36843991038629"],[1589887094,"56.49640324593766"],[1589893574,"56.61097964792364"],[1589900054,"64.06907657719732"],[1589906534,"64.38918178274582"],[1589913014,"64.50178951803102"],[1589919494,"64.4053248496923"],[1589925974,"58.69304155067939"],[1589932454,"58.994247555900294"],[1589938934,"59.14819729190208"],[1589945414,"59.275373160773135"],[1589951894,"59.25489902708492"],[1589958374,"55.760121899842105"],[1589964854,"56.091645372255186"],[1589971334,"56.26961284200662"],[1589977814,"56.78304111757271"],[1589984294,"62.006701341449485"],[1589990774,"62.4720941495163"],[1589997254,"62.69061615330401"],[1590003734,"62.80558628863017"],[1590010214,"62.31026974671134"],[1590016694,"62.344918280645246"],[1590023174,"62.51304241689274"],[1590029654,"62.64691175254647"],[1590036134,"62.794168021765586"],[1590042614,"58.300489410541815"],[1590049094,"56.32158564290748"],[1590055574,"56.41175057780367"],[1590062054,"56.42907484477063"],[1590068534,"66.87127675910213"],[1590075014,"63.41547923253498"],[1590081494,"63.612345902613995"],[1590087974,"63.79070710570559"],[1590094454,"63.9848176424035"],[1590100934,"64.13758617838484"],[1590107414,"64.44784805042937"],[1590113894,"64.66794498757771"],[1590120374,"64.92544659204107"],[1590126854,"65.04041672736722"],[1590133334,"55.870760968426524"],[1590139814,"56.92006031994771"],[1590146294,"57.03778658865497"],[1590152774,"56.927147520070555"],[1590159254,"65.77591060678245"],[1590165734,"66.05349261159387"],[1590172214,"66.13184554628533"],[1590178694,"65.58101260340422"],[1590185174,"65.7077947389351"],[1590191654,"65.77039834002024"],[1590198134,"65.81843380751951"],[1590204614,"65.89678674221096"],[1590211094,"65.86450060831801"],[1590217574,"55.855799101500516"],[1590224054,"56.03888510467401"],[1590230534,"56.249532441658566"],[1590237014,"56.449942711799004"],[1590243494,"65.71094460565638"],[1590249974,"65.75543647309424"],[1590256454,"65.91135487579682"],[1590262934,"60.49279664854181"],[1590269414,"60.8298323877171"],[1590275894,"60.95976438996925"],[1590282374,"61.06764732517256"],[1590288854,"61.4416939983227"],[1590295334,"61.58343800077959"],[1590301814,"58.251272743022064"],[1590308294,"55.515613495603965"],[1590314774,"56.27040030868694"],[1590321254,"56.40426964434068"],[1590327734,"65.59597447033022"],[1590334214,"66.10349674579395"],[1590340694,"65.84875127471169"],[1590347174,"66.1133400792979"],[1590353654,"66.26886474866033"],[1590360134,"66.2728020820619"]]}]}}
|
|
|
|
|
|
{"status":"success","data":{"resultType":"matrix","result":[{"metric":{},"values":[[1587768134,"51.542056626728986"],[1587774614,"52.020442635021006"],[1587781094,"52.37834624122467"],[1587787574,"52.76066131451813"],[1587794054,"53.076435453324876"],[1587800534,"44.07096649723009"],[1587807014,"44.70212104150343"],[1587813494,"45.22263651719236"],[1587819974,"45.74512065958209"],[1587826454,"53.23471625606842"],[1587832934,"53.66861039692258"],[1587839414,"53.965879068741906"],[1587845894,"54.20763133959894"],[1587852374,"52.61891731206123"],[1587858854,"53.1362829210289"],[1587865334,"53.43512652620886"],[1587871814,"53.82925359970706"],[1587878294,"54.01745813630261"],[1587884774,"46.972387480854714"],[1587891254,"47.647246425885605"],[1587897734,"47.91813496391434"],[1587904214,"48.1783927017588"],[1587910694,"56.29796164249801"],[1587917174,"55.89674736887695"],[1587923654,"56.29402430909642"],[1587930134,"56.56136924706373"],[1587936614,"56.62791018155045"],[1587943094,"56.18732257391359"],[1587949574,"56.39285137747609"],[1587956054,"55.85540536816036"],[1587962534,"56.63735978171424"],[1587969014,"52.26101370585757"],[1587975494,"49.91672539855657"],[1587981974,"50.097842735029275"],[1587988454,"51.35818315687517"],[1587994934,"57.60869993188413"],[1588001414,"57.74453793423866"],[1588007894,"58.00203953870202"],[1588014374,"58.25639127644412"],[1588020854,"58.43239007949476"],[1588027334,"58.66863008358959"],[1588033814,"59.46042783064741"],[1588040294,"59.706117434906034"],[1588046774,"59.742340902200574"],[1588053254,"59.74745943562263"],[1588059734,"51.33455915646569"],[1588066214,"51.8243634316223"],[1588072694,"52.80594064863631"],[1588079174,"52.968552518121584"],[1588085654,"60.73258025269806"],[1588092134,"60.15930450942795"],[1588098614,"60.27978691151631"],[1588105094,"60.449092247784264"],[1588111574,"60.51090838218908"],[1588118054,"60.247894510963505"],[1588124534,"56.15936750676237"],[1588131014,"56.37828324389025"],[1588137494,"55.933758302851814"],[1588143974,"51.0483150181708"],[1588150454,"51.368813957059444"],[1588156934,"51.647577161891334"],[1588163414,"52.327554640344275"],[1588169894,"58.49971848066179"],[1588176374,"58.60130168242256"],[1588182854,"59.11472995798866"],[1588189334,"59.69273050134066"],[1588195814,"59.70218010150445"],[1588202294,"59.61083396658778"],[1588208774,"60.3727079797936"],[1588215254,"60.589261316880524"],[1588221734,"60.67824505175625"],[1588228214,"57.0074691214628"],[1588234694,"52.60553037849586"],[1588241174,"52.750030514333865"],[1588247654,"52.844920249311954"],[1588254134,"59.189539292618676"],[1588260614,"59.525393831773485"],[1588267094,"59.73249756869663"],[1588273574,"59.887234771378736"],[1588280054,"60.09000744156013"],[1588286534,"60.28529917827852"],[1588293014,"55.68767496525303"],[1588299494,"56.61531071466539"],[1588305974,"56.97754538761078"],[1588312454,"57.08070352273219"],[1588318934,"52.54292677741073"],[1588325414,"52.75042424767402"],[1588331894,"52.92484811736403"],[1588338374,"53.02997491918623"],[1588344854,"65.66724020489883"],[1588351334,"65.79835340717146"],[1588357814,"65.3577657995346"],[1588364294,"59.78683276963843"],[1588370774,"60.39200091346135"],[1588377254,"60.567212249831684"],[1588383734,"61.894487339504444"],[1588390214,"62.03308147524008"],[1588396694,"61.97559640757701"],[1588403174,"53.87610786718587"],[1588409654,"54.12219120478464"],[1588416134,"54.370243209084215"],[1588422614,"54.50214387803716"],[1588429094,"62.339406013883035"],[1588435574,"59.50255729804432"],[1588442054,"59.81360663676918"],[1588448534,"59.98488063973794"],[1588455014,"60.113631441969616"],[1588461494,"60.255375444426505"],[1588467974,"60.48649691509927"],[1588474454,"60.56524358313089"],[1588480934,"60.67470145169482"],[1588487414,"57.417345528567324"],[1588493894,"54.637981880391685"],[1588500374,"54.83287988376991"],[1588506854,"55.1990518901169"],[1588513334,"62.47721268293836"],[1588519814,"63.006784025450926"],[1588526294,"63.28475976360251"],[1588532774,"63.351300698089204"],[1588539254,"63.70211710417003"],[1588545734,"63.82417443961902"],[1588552214,"63.90961457443331"],[1588558694,"64.04624004346816"],[1588565174,"64.16239137881479"],[1588571654,"64.24546911358813"],[1588578134,"55.72114229916647"],[1588584614,"55.98888097047394"],[1588591094,"56.145980573196994"],[1588597574,"56.13692470637336"],[1588604054,"62.8953574901862"],[1588610534,"62.59178908492434"],[1588617014,"62.746132554266296"],[1588623494,"62.88236428996098"],[1588629974,"63.34027616456479"],[1588636454,"63.80685017265208"],[1588642934,"64.1312864449423"],[1588649414,"64.40217498297103"],[1588655894,"64.66479512085644"],[1588662374,"56.281031108871204"],[1588668854,"56.79367191775698"],[1588675334,"56.839738718555466"],[1588681814,"57.234259525393824"],[1588688294,"65.77433567342182"],[1588694774,"65.76803593997928"],[1588701254,"65.82079620756048"],[1588707734,"66.50510475275514"],[1588714214,"66.48069328566535"],[1588720694,"66.31020674937692"],[1588727174,"64.84236885726773"],[1588733654,"64.64707712054934"],[1588740134,"64.82228845691967"],[1588746614,"62.07875454269841"],[1588753094,"55.63215856429075"],[1588759574,"55.6955496320562"],[1588766054,"55.75775949980116"],[1588772534,"63.28987829702456"],[1588779014,"63.52729950113986"],[1588785494,"63.93441977486327"],[1588791974,"64.12774284488087"],[1588798454,"64.70534965489273"],[1588804934,"57.931561270813724"],[1588811414,"57.955579004563376"],[1588817894,"58.00715807212408"],[1588824374,"58.12173447411006"],[1588830854,"58.300489410541815"],[1588837334,"55.20259549017832"],[1588843814,"56.10227617243946"],[1588850294,"56.15661137338126"],[1588856774,"56.22393977454829"],[1588863254,"63.04891349284784"],[1588869734,"64.34350871528748"],[1588876214,"63.96040617531371"],[1588882694,"64.19192137932663"],[1588889174,"64.09309431094697"],[1588895654,"64.25688738045271"],[1588902134,"63.71944137113699"],[1588908614,"63.92497017469948"],[1588915094,"64.12104937809819"],[1588921574,"55.9479327030975"],[1588928054,"55.649876564597854"],[1588934534,"55.750672299678314"],[1588941014,"55.86367376830368"],[1588947494,"62.53824135066285"],[1588953974,"62.677622953078796"],[1588960454,"62.461463349332035"],[1588966934,"63.046157359466726"],[1588973414,"63.60525870249115"],[1588979894,"63.746608971607884"],[1588986374,"64.45336031719158"],[1588992854,"64.54549391878857"],[1588999334,"63.999385775989346"],[1589005814,"59.296241027801514"],[1589012294,"56.45033644513916"],[1589018774,"56.51963351300699"],[1589025254,"56.64680938187803"],[1589031734,"64.08403844412334"],[1589038214,"64.1950712460479"],[1589044694,"64.24861898030939"],[1589051174,"60.40145051362514"],[1589057654,"60.43491784753857"],[1589064134,"60.52941384917651"],[1589070614,"60.711318652329524"],[1589077094,"60.75620425310754"],[1589083574,"60.88731745538018"],[1589090054,"60.956220789907825"],[1589096534,"56.18653510723327"],[1589103014,"56.94801538709894"],[1589109494,"57.14094472377638"],[1589115974,"57.29883179317975"],[1589122454,"64.34468991530797"],[1589128934,"64.43367365018368"],[1589135414,"64.57226778591931"],[1589141894,"64.80023938987081"],[1589148374,"64.9758444595813"],[1589154854,"65.16523019619733"],[1589161334,"65.20657219691391"],[1589167814,"65.38847700006694"],[1589174294,"65.46171140133633"],[1589180774,"55.98888097047394"],[1589187254,"56.30071777587911"],[1589193734,"56.54798231349836"],[1589200214,"57.264576992586"],[1589206694,"63.6985735041086"],[1589213174,"63.1619149614732"],[1589219654,"63.45327763319014"],[1589226134,"63.68558030388339"],[1589232614,"63.87063497375768"],[1589239094,"64.05096484355006"],[1589245574,"65.18412939652491"],[1589252054,"64.95852019261436"],[1589258534,"64.82898192370236"],[1589265014,"60.18765330991932"],[1589271494,"56.716500183086005"],[1589277974,"56.7306745833317"],[1589284454,"56.94959032045956"],[1589290934,"64.22853857996134"],[1589297414,"59.61634623335"],[1589303894,"60.08370770811759"],[1589310374,"60.19316557668154"],[1589316854,"58.702884884183334"],[1589323334,"59.011571822867246"],[1589329814,"59.25962382716682"],[1589336294,"59.39821796290244"],[1589342774,"59.480508230995476"],[1589349254,"59.56279849908851"],[1589355734,"55.39316242681481"],[1589362214,"55.45379736119915"],[1589368694,"55.59829749703715"],[1589375174,"55.74122269951452"],[1589381654,"63.815512306135545"],[1589388134,"64.00332310939093"],[1589394614,"58.54893514818155"],[1589401094,"59.167096492229675"],[1589407574,"59.240724626839224"],[1589414054,"59.47617716425374"],[1589420534,"60.130168242256254"],[1589427014,"59.877391437874785"],[1589433494,"59.98094330633636"],[1589439974,"56.420018977946995"],[1589446454,"56.794065651097135"],[1589452934,"56.97557672090999"],[1589459414,"56.931084853472136"],[1589465894,"58.31702621082846"],[1589472374,"58.846203820000866"],[1589478854,"58.95408675520417"],[1589485334,"59.17221502565173"],[1589491814,"59.3714440957717"],[1589498294,"59.584060099457034"],[1589504774,"59.792345036400654"],[1589511254,"59.908102638407115"],[1589517734,"60.09512597498219"],[1589524214,"59.02732115647357"],[1589530694,"56.607829781202376"],[1589537174,"56.6995696494592"],[1589543654,"56.83501391847358"],[1589550134,"63.209556695632315"],[1589556614,"63.340669897904945"],[1589563094,"63.504069234070535"],[1589569574,"63.65683777005186"],[1589576054,"63.94820044176881"],[1589582534,"63.99505470924761"],[1589589014,"64.5423440520673"],[1589595494,"64.98371912638447"],[1589601974,"65.10695766185394"],[1589608454,"65.57195673658059"],[1589614934,"55.69594336539635"],[1589621414,"55.97313163686761"],[1589627894,"56.786190984293974"],[1589634374,"56.817295918166465"],[1589640854,"61.338142129861126"],[1589647334,"61.527921599817304"],[1589653814,"61.64486040184425"],[1589660294,"61.77833600415782"],[1589666774,"61.821646671575216"],[1589673254,"61.968902940794315"],[1589679734,"62.21931734513484"],[1589686214,"62.27365254607664"],[1589692694,"62.38822894806264"],[1589699174,"55.683343898511296"],[1589705654,"56.56688151382595"],[1589712134,"56.67437071568909"],[1589718614,"56.70547564956158"],[1589725094,"64.42422405001989"],[1589731574,"64.55297485225157"],[1589738054,"64.66558258753676"],[1589744534,"64.83921899054647"],[1589751014,"64.98883765980652"],[1589757494,"65.08608979482555"],[1589763974,"65.423519267341"],[1589770454,"65.5924308702688"],[1589776934,"65.77906047350372"],[1589783414,"63.95331897519087"],[1589789894,"56.08967670555439"],[1589796374,"56.26410057524441"],[1589802854,"56.41726284456588"],[1589809334,"63.49343843388626"],[1589815814,"63.809606306033174"],[1589822294,"61.586981600841014"],[1589828774,"61.63737946838125"],[1589835254,"61.97914000763842"],[1589841734,"62.15789494407018"],[1589848214,"62.45083254914776"],[1589854694,"62.653211485989004"],[1589861174,"62.82251682225696"],[1589867654,"62.358698947550785"],[1589874134,"55.99084963717472"],[1589880614,"56.36843991038629"],[1589887094,"56.49640324593766"],[1589893574,"56.61097964792364"],[1589900054,"64.06907657719732"],[1589906534,"64.38918178274582"],[1589913014,"64.50178951803102"],[1589919494,"64.4053248496923"],[1589925974,"58.69304155067939"],[1589932454,"58.994247555900294"],[1589938934,"59.14819729190208"],[1589945414,"59.275373160773135"],[1589951894,"59.25489902708492"],[1589958374,"55.760121899842105"],[1589964854,"56.091645372255186"],[1589971334,"56.26961284200662"],[1589977814,"56.78304111757271"],[1589984294,"62.006701341449485"],[1589990774,"62.4720941495163"],[1589997254,"62.69061615330401"],[1590003734,"62.80558628863017"],[1590010214,"62.31026974671134"],[1590016694,"62.344918280645246"],[1590023174,"62.51304241689274"],[1590029654,"62.64691175254647"],[1590036134,"62.794168021765586"],[1590042614,"58.300489410541815"],[1590049094,"56.32158564290748"],[1590055574,"56.41175057780367"],[1590062054,"56.42907484477063"],[1590068534,"66.87127675910213"],[1590075014,"63.41547923253498"],[1590081494,"63.612345902613995"],[1590087974,"63.79070710570559"],[1590094454,"63.9848176424035"],[1590100934,"64.13758617838484"],[1590107414,"64.44784805042937"],[1590113894,"64.66794498757771"],[1590120374,"64.92544659204107"],[1590126854,"65.04041672736722"],[1590133334,"55.870760968426524"],[1590139814,"56.92006031994771"],[1590146294,"57.03778658865497"],[1590152774,"56.927147520070555"],[1590159254,"65.77591060678245"],[1590165734,"66.05349261159387"],[1590172214,"66.13184554628533"],[1590178694,"65.58101260340422"],[1590185174,"65.7077947389351"],[1590191654,"65.77039834002024"],[1590198134,"65.81843380751951"],[1590204614,"65.89678674221096"],[1590211094,"65.86450060831801"],[1590217574,"55.855799101500516"],[1590224054,"56.03888510467401"],[1590230534,"56.249532441658566"],[1590237014,"56.449942711799004"],[1590243494,"65.71094460565638"],[1590249974,"65.75543647309424"],[1590256454,"65.91135487579682"],[1590262934,"60.49279664854181"],[1590269414,"60.8298323877171"],[1590275894,"60.95976438996925"],[1590282374,"61.06764732517256"],[1590288854,"61.4416939983227"],[1590295334,"61.58343800077959"],[1590301814,"58.251272743022064"],[1590308294,"55.515613495603965"],[1590314774,"56.27040030868694"],[1590321254,"56.40426964434068"],[1590327734,"65.59597447033022"],[1590334214,"66.10349674579395"],[1590340694,"65.84875127471169"],[1590347174,"66.1133400792979"],[1590353654,"66.26886474866033"],[1590360134,"66.2728020820619"]]}]}}
|
|
|
|
*/
|
|
|
|
//This will get triggered roughly every minute (10 seconds in debug), but we don't want to healthcheck that frequently
|
|
if (!DateUtil.IsAfterDuration(lastServerCheckDone, DO_EVERY_INTERVAL))
|
|
return;
|
|
|
|
log.LogTrace("Starting metrics snapshot");
|
|
|
|
//Gather core metrics here
|
|
var process = Process.GetCurrentProcess();
|
|
|
|
//PHYSICAL MEMORY
|
|
// metrics.Measure.Gauge.SetValue(MetricsRegistry.PhysicalMemoryGauge, process.WorkingSet64);
|
|
|
|
//PRIVATE BYTES
|
|
// metrics.Measure.Gauge.SetValue(MetricsRegistry.PrivateBytesGauge, process.PrivateMemorySize64);
|
|
|
|
//RECORDS IN TABLE
|
|
//Only do this once per hour
|
|
if (DateUtil.IsAfterDuration(lastRecordCountCheck, 1))
|
|
{
|
|
lastRecordCountCheck = DateTime.UtcNow;
|
|
log.LogTrace("Counting table records");
|
|
|
|
|
|
//Get a count of important tables in db
|
|
List<string> allTableNames = await DbUtil.GetAllTablenamesAsync();
|
|
|
|
//Skip some tables as they are internal and / or only ever have one record
|
|
List<string> skipTableNames = new List<string>();
|
|
skipTableNames.Add("alicense");
|
|
skipTableNames.Add("aschemaversion");
|
|
|
|
foreach (string table in allTableNames)
|
|
{
|
|
if (!skipTableNames.Contains(table))
|
|
{
|
|
//var tags = new MetricTags("TableTagKey", table);
|
|
// metrics.Measure.Gauge.SetValue(MetricsRegistry.DBRecordsGauge, tags, await DbUtil.CountOfRecordsAsync(table));
|
|
}
|
|
}
|
|
}
|
|
|
|
//JOB COUNTS (DEAD, RUNNING, COMPLETED, SLEEPING)
|
|
|
|
foreach (JobStatus stat in Enum.GetValues(typeof(JobStatus)))
|
|
{
|
|
// var jobtag = new MetricTags("JobStatus", stat.ToString());
|
|
// metrics.Measure.Gauge.SetValue(MetricsRegistry.JobsGauge, jobtag, await JobsBiz.GetCountForJobStatusAsync(ct, stat));
|
|
}
|
|
|
|
|
|
|
|
//FILES ON DISK
|
|
//Only do this once per hour
|
|
if (DateUtil.IsAfterDuration(lastFileCountCheck, 1))
|
|
{
|
|
lastFileCountCheck = DateTime.UtcNow;
|
|
log.LogTrace("Files on disk information");
|
|
var UtilFilesInfo = FileUtil.GetUtilityFolderSizeInfo();
|
|
var UserFilesInfo = FileUtil.GetAttachmentFolderSizeInfo();
|
|
|
|
// var mtag = new MetricTags("File type", "Business object files");
|
|
// metrics.Measure.Gauge.SetValue(MetricsRegistry.FileCountGauge, mtag, UserFilesInfo.FileCountWithChildren);
|
|
// metrics.Measure.Gauge.SetValue(MetricsRegistry.FileSizeGauge, mtag, UserFilesInfo.SizeWithChildren);
|
|
|
|
// mtag = new MetricTags("File type", "OPS files");
|
|
// metrics.Measure.Gauge.SetValue(MetricsRegistry.FileCountGauge, mtag, UtilFilesInfo.FileCountWithChildren);
|
|
// metrics.Measure.Gauge.SetValue(MetricsRegistry.FileSizeGauge, mtag, UtilFilesInfo.SizeWithChildren);
|
|
|
|
}
|
|
|
|
|
|
lastServerCheckDone = DateTime.UtcNow;
|
|
|
|
|
|
//just to hide compiler warning for now
|
|
await Task.CompletedTask;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/////////////////////////////////////////////////////////////////////
|
|
|
|
}//eoc
|
|
|
|
|
|
}//eons
|
|
|