diff --git a/.vscode/launch.json b/.vscode/launch.json index e3a31ed6..6b9a0d68 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -50,8 +50,8 @@ "AYANOVA_FOLDER_USER_FILES": "c:\\temp\\RavenTestData\\userfiles", "AYANOVA_FOLDER_BACKUP_FILES": "c:\\temp\\RavenTestData\\backupfiles", "AYANOVA_METRICS_USE_INFLUXDB": "false", - "AYANOVA_SERVER_TEST_MODE":"false", - "AYANOVA_SERVER_TEST_MODE_SEEDLEVEL":"huge", + "AYANOVA_SERVER_TEST_MODE":"true", + "AYANOVA_SERVER_TEST_MODE_SEEDLEVEL":"small", "AYANOVA_SERVER_TEST_MODE_TZ_OFFSET":"-7", "AYANOVA_BACKUP_PG_DUMP_PATH":"C:\\data\\code\\PostgreSQLPortable_12.0\\App\\PgSQL\\bin\\" diff --git a/server/AyaNova/biz/JobsBiz.cs b/server/AyaNova/biz/JobsBiz.cs index 6a645cfe..8cfbc6b0 100644 --- a/server/AyaNova/biz/JobsBiz.cs +++ b/server/AyaNova/biz/JobsBiz.cs @@ -364,6 +364,8 @@ namespace AyaNova.Biz //backup await CoreJobBackup.DoWorkAsync(ct); + await CoreJobMetricsSnapshot.DoJobAsync(ct); + //Notifications } diff --git a/server/AyaNova/generator/CoreJobMetricsSnapshot.cs b/server/AyaNova/generator/CoreJobMetricsSnapshot.cs index fca38da7..12512c3c 100644 --- a/server/AyaNova/generator/CoreJobMetricsSnapshot.cs +++ b/server/AyaNova/generator/CoreJobMetricsSnapshot.cs @@ -1,132 +1,193 @@ -// using System; -// using System.Collections.Generic; -// using System.Threading.Tasks; -// using System.Diagnostics; -// using Microsoft.Extensions.Logging; -// using App.Metrics; -// using AyaNova.Util; -// using AyaNova.Models; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using System.Diagnostics; +using Microsoft.Extensions.Logging; +using AyaNova.Util; +using AyaNova.Models; -// namespace AyaNova.Biz -// { +namespace AyaNova.Biz +{ + /// + /// called by Generator to gather server metrics and insert in db + /// + internal static class CoreJobMetricsSnapshot + { + private static ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("CoreJobMetricsSnapshot"); + +#if (DEBUG) + private static TimeSpan DO_EVERY_INTERVAL = new TimeSpan(0, 1, 0);//DEBUG do a check every 60 seconds +#else + private static TimeSpan DO_EVERY_INTERVAL = new TimeSpan(0, 15, 0);//RELEASE do a check every 15 minutes +#endif + + private static DateTime lastServerCheckDone = DateTime.MinValue; + private static DateTime lastRecordCountCheck = DateTime.MinValue; + private static DateTime lastFileCountCheck = DateTime.MinValue; + + //////////////////////////////////////////////////////////////////////////////////////////////// + // DoAsync + // + public static async Task DoJobAsync(AyContext ct) + { + //Gather stats, output to database but only every minute or more + /* + //TODO: figure out teh best format to store it in based on what I need at the client end + // retention setting defaults to 1 year? + Generate sample data, see how large it would be under various scenarios + i.e. if I gather every minute, how much can I practically store? + Shortest time frame that is shown for DO is 1 minute intervals (6 hour / 360 entries) + Timestamp, decimal/float value (need to determine this number type) + Make a chart at client with test data from digital ocean to play with + try to replicate their stuff to learn how to best do it + Downsampling, should I convert old data to downsampled so I can keep more of it or is this presentation only + depends on storage space I guess + seperate tables per metric? + Seems likely since there would be less data to move around, but if I'm fetching all anyway?? + hmm... something to consider / experiment with + + Some D.O. charts have max value at top of left axis (y?) that change with the range and values and some have 100% or 1.0 and never change that axis + + + + + + + Digital ocean has a seperate query for each type of info + Implying maybe seperate storage for each type which makes sense I guess + though their data comes from all over the place + + For a 30 day period their data came across like this: + 109 minutes apart + CPU average: 396 entries 1.8 times per hour or every 109 minutes + Disk usage: 400 entries every 108 minutes (peculiar) + + For 6 hour period get 360 entries?! + 1 minute apart + they are all today, so it's always returning the same number of entries, regardless of time period + they are one minute apart + so it must be sampling somehow for the longer periods at least + + for 24 hours get 401 entries + 3.6 minutes apart + + 7 days get 400 entries + 25 minutes apart -// /// -// /// called by Generator to gather server metrics and check on things -// /// See MetricsRegistry for defined metrics -// /// -// /// -// internal static class CoreJobMetricsSnapshot -// { -// private static ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("CoreJobMetricsSnapshot"); + OK, not a trivial thing to graph it requires "downsampling" or "decimation" + Here is a link to a c# method that will accomplish this: + https://github.com/sveinn-steinarsson/flot-downsample/ + c# + https://gist.github.com/DanielWJudge/63300889f27c7f50eeb7 -// #if (DEBUG) -// private static TimeSpan DO_EVERY_INTERVAL = new TimeSpan(0, 1, 0);//DEBUG do a check every 60 seconds -// #else -// private static TimeSpan DO_EVERY_INTERVAL = new TimeSpan(0, 15, 0);//RELEASE do a check every 15 minutes -// #endif + + CREATE flat_table( + trip_id integer, + tstamp timestamptz, + speed float, + distance float, + temperature float, + ,...); -// private static DateTime lastServerCheckDone = DateTime.MinValue; -// private static DateTime lastRecordCountCheck = DateTime.MinValue; -// private static DateTime lastFileCountCheck = DateTime.MinValue; + actual data: -// //////////////////////////////////////////////////////////////////////////////////////////////// -// // DoAsync -// // -// public static async Task DoJobAsync(AyContext ct) -// { -// //https://www.app-metrics.io/ -// IMetrics metrics = (IMetrics)ServiceProviderProvider.Provider.GetService(typeof(IMetrics)); + {"status":"success","data":{"resultType":"matrix","result":[{"metric":{},"values":[[1587768134,"51.542056626728986"],[1587774614,"52.020442635021006"],[1587781094,"52.37834624122467"],[1587787574,"52.76066131451813"],[1587794054,"53.076435453324876"],[1587800534,"44.07096649723009"],[1587807014,"44.70212104150343"],[1587813494,"45.22263651719236"],[1587819974,"45.74512065958209"],[1587826454,"53.23471625606842"],[1587832934,"53.66861039692258"],[1587839414,"53.965879068741906"],[1587845894,"54.20763133959894"],[1587852374,"52.61891731206123"],[1587858854,"53.1362829210289"],[1587865334,"53.43512652620886"],[1587871814,"53.82925359970706"],[1587878294,"54.01745813630261"],[1587884774,"46.972387480854714"],[1587891254,"47.647246425885605"],[1587897734,"47.91813496391434"],[1587904214,"48.1783927017588"],[1587910694,"56.29796164249801"],[1587917174,"55.89674736887695"],[1587923654,"56.29402430909642"],[1587930134,"56.56136924706373"],[1587936614,"56.62791018155045"],[1587943094,"56.18732257391359"],[1587949574,"56.39285137747609"],[1587956054,"55.85540536816036"],[1587962534,"56.63735978171424"],[1587969014,"52.26101370585757"],[1587975494,"49.91672539855657"],[1587981974,"50.097842735029275"],[1587988454,"51.35818315687517"],[1587994934,"57.60869993188413"],[1588001414,"57.74453793423866"],[1588007894,"58.00203953870202"],[1588014374,"58.25639127644412"],[1588020854,"58.43239007949476"],[1588027334,"58.66863008358959"],[1588033814,"59.46042783064741"],[1588040294,"59.706117434906034"],[1588046774,"59.742340902200574"],[1588053254,"59.74745943562263"],[1588059734,"51.33455915646569"],[1588066214,"51.8243634316223"],[1588072694,"52.80594064863631"],[1588079174,"52.968552518121584"],[1588085654,"60.73258025269806"],[1588092134,"60.15930450942795"],[1588098614,"60.27978691151631"],[1588105094,"60.449092247784264"],[1588111574,"60.51090838218908"],[1588118054,"60.247894510963505"],[1588124534,"56.15936750676237"],[1588131014,"56.37828324389025"],[1588137494,"55.933758302851814"],[1588143974,"51.0483150181708"],[1588150454,"51.368813957059444"],[1588156934,"51.647577161891334"],[1588163414,"52.327554640344275"],[1588169894,"58.49971848066179"],[1588176374,"58.60130168242256"],[1588182854,"59.11472995798866"],[1588189334,"59.69273050134066"],[1588195814,"59.70218010150445"],[1588202294,"59.61083396658778"],[1588208774,"60.3727079797936"],[1588215254,"60.589261316880524"],[1588221734,"60.67824505175625"],[1588228214,"57.0074691214628"],[1588234694,"52.60553037849586"],[1588241174,"52.750030514333865"],[1588247654,"52.844920249311954"],[1588254134,"59.189539292618676"],[1588260614,"59.525393831773485"],[1588267094,"59.73249756869663"],[1588273574,"59.887234771378736"],[1588280054,"60.09000744156013"],[1588286534,"60.28529917827852"],[1588293014,"55.68767496525303"],[1588299494,"56.61531071466539"],[1588305974,"56.97754538761078"],[1588312454,"57.08070352273219"],[1588318934,"52.54292677741073"],[1588325414,"52.75042424767402"],[1588331894,"52.92484811736403"],[1588338374,"53.02997491918623"],[1588344854,"65.66724020489883"],[1588351334,"65.79835340717146"],[1588357814,"65.3577657995346"],[1588364294,"59.78683276963843"],[1588370774,"60.39200091346135"],[1588377254,"60.567212249831684"],[1588383734,"61.894487339504444"],[1588390214,"62.03308147524008"],[1588396694,"61.97559640757701"],[1588403174,"53.87610786718587"],[1588409654,"54.12219120478464"],[1588416134,"54.370243209084215"],[1588422614,"54.50214387803716"],[1588429094,"62.339406013883035"],[1588435574,"59.50255729804432"],[1588442054,"59.81360663676918"],[1588448534,"59.98488063973794"],[1588455014,"60.113631441969616"],[1588461494,"60.255375444426505"],[1588467974,"60.48649691509927"],[1588474454,"60.56524358313089"],[1588480934,"60.67470145169482"],[1588487414,"57.417345528567324"],[1588493894,"54.637981880391685"],[1588500374,"54.83287988376991"],[1588506854,"55.1990518901169"],[1588513334,"62.47721268293836"],[1588519814,"63.006784025450926"],[1588526294,"63.28475976360251"],[1588532774,"63.351300698089204"],[1588539254,"63.70211710417003"],[1588545734,"63.82417443961902"],[1588552214,"63.90961457443331"],[1588558694,"64.04624004346816"],[1588565174,"64.16239137881479"],[1588571654,"64.24546911358813"],[1588578134,"55.72114229916647"],[1588584614,"55.98888097047394"],[1588591094,"56.145980573196994"],[1588597574,"56.13692470637336"],[1588604054,"62.8953574901862"],[1588610534,"62.59178908492434"],[1588617014,"62.746132554266296"],[1588623494,"62.88236428996098"],[1588629974,"63.34027616456479"],[1588636454,"63.80685017265208"],[1588642934,"64.1312864449423"],[1588649414,"64.40217498297103"],[1588655894,"64.66479512085644"],[1588662374,"56.281031108871204"],[1588668854,"56.79367191775698"],[1588675334,"56.839738718555466"],[1588681814,"57.234259525393824"],[1588688294,"65.77433567342182"],[1588694774,"65.76803593997928"],[1588701254,"65.82079620756048"],[1588707734,"66.50510475275514"],[1588714214,"66.48069328566535"],[1588720694,"66.31020674937692"],[1588727174,"64.84236885726773"],[1588733654,"64.64707712054934"],[1588740134,"64.82228845691967"],[1588746614,"62.07875454269841"],[1588753094,"55.63215856429075"],[1588759574,"55.6955496320562"],[1588766054,"55.75775949980116"],[1588772534,"63.28987829702456"],[1588779014,"63.52729950113986"],[1588785494,"63.93441977486327"],[1588791974,"64.12774284488087"],[1588798454,"64.70534965489273"],[1588804934,"57.931561270813724"],[1588811414,"57.955579004563376"],[1588817894,"58.00715807212408"],[1588824374,"58.12173447411006"],[1588830854,"58.300489410541815"],[1588837334,"55.20259549017832"],[1588843814,"56.10227617243946"],[1588850294,"56.15661137338126"],[1588856774,"56.22393977454829"],[1588863254,"63.04891349284784"],[1588869734,"64.34350871528748"],[1588876214,"63.96040617531371"],[1588882694,"64.19192137932663"],[1588889174,"64.09309431094697"],[1588895654,"64.25688738045271"],[1588902134,"63.71944137113699"],[1588908614,"63.92497017469948"],[1588915094,"64.12104937809819"],[1588921574,"55.9479327030975"],[1588928054,"55.649876564597854"],[1588934534,"55.750672299678314"],[1588941014,"55.86367376830368"],[1588947494,"62.53824135066285"],[1588953974,"62.677622953078796"],[1588960454,"62.461463349332035"],[1588966934,"63.046157359466726"],[1588973414,"63.60525870249115"],[1588979894,"63.746608971607884"],[1588986374,"64.45336031719158"],[1588992854,"64.54549391878857"],[1588999334,"63.999385775989346"],[1589005814,"59.296241027801514"],[1589012294,"56.45033644513916"],[1589018774,"56.51963351300699"],[1589025254,"56.64680938187803"],[1589031734,"64.08403844412334"],[1589038214,"64.1950712460479"],[1589044694,"64.24861898030939"],[1589051174,"60.40145051362514"],[1589057654,"60.43491784753857"],[1589064134,"60.52941384917651"],[1589070614,"60.711318652329524"],[1589077094,"60.75620425310754"],[1589083574,"60.88731745538018"],[1589090054,"60.956220789907825"],[1589096534,"56.18653510723327"],[1589103014,"56.94801538709894"],[1589109494,"57.14094472377638"],[1589115974,"57.29883179317975"],[1589122454,"64.34468991530797"],[1589128934,"64.43367365018368"],[1589135414,"64.57226778591931"],[1589141894,"64.80023938987081"],[1589148374,"64.9758444595813"],[1589154854,"65.16523019619733"],[1589161334,"65.20657219691391"],[1589167814,"65.38847700006694"],[1589174294,"65.46171140133633"],[1589180774,"55.98888097047394"],[1589187254,"56.30071777587911"],[1589193734,"56.54798231349836"],[1589200214,"57.264576992586"],[1589206694,"63.6985735041086"],[1589213174,"63.1619149614732"],[1589219654,"63.45327763319014"],[1589226134,"63.68558030388339"],[1589232614,"63.87063497375768"],[1589239094,"64.05096484355006"],[1589245574,"65.18412939652491"],[1589252054,"64.95852019261436"],[1589258534,"64.82898192370236"],[1589265014,"60.18765330991932"],[1589271494,"56.716500183086005"],[1589277974,"56.7306745833317"],[1589284454,"56.94959032045956"],[1589290934,"64.22853857996134"],[1589297414,"59.61634623335"],[1589303894,"60.08370770811759"],[1589310374,"60.19316557668154"],[1589316854,"58.702884884183334"],[1589323334,"59.011571822867246"],[1589329814,"59.25962382716682"],[1589336294,"59.39821796290244"],[1589342774,"59.480508230995476"],[1589349254,"59.56279849908851"],[1589355734,"55.39316242681481"],[1589362214,"55.45379736119915"],[1589368694,"55.59829749703715"],[1589375174,"55.74122269951452"],[1589381654,"63.815512306135545"],[1589388134,"64.00332310939093"],[1589394614,"58.54893514818155"],[1589401094,"59.167096492229675"],[1589407574,"59.240724626839224"],[1589414054,"59.47617716425374"],[1589420534,"60.130168242256254"],[1589427014,"59.877391437874785"],[1589433494,"59.98094330633636"],[1589439974,"56.420018977946995"],[1589446454,"56.794065651097135"],[1589452934,"56.97557672090999"],[1589459414,"56.931084853472136"],[1589465894,"58.31702621082846"],[1589472374,"58.846203820000866"],[1589478854,"58.95408675520417"],[1589485334,"59.17221502565173"],[1589491814,"59.3714440957717"],[1589498294,"59.584060099457034"],[1589504774,"59.792345036400654"],[1589511254,"59.908102638407115"],[1589517734,"60.09512597498219"],[1589524214,"59.02732115647357"],[1589530694,"56.607829781202376"],[1589537174,"56.6995696494592"],[1589543654,"56.83501391847358"],[1589550134,"63.209556695632315"],[1589556614,"63.340669897904945"],[1589563094,"63.504069234070535"],[1589569574,"63.65683777005186"],[1589576054,"63.94820044176881"],[1589582534,"63.99505470924761"],[1589589014,"64.5423440520673"],[1589595494,"64.98371912638447"],[1589601974,"65.10695766185394"],[1589608454,"65.57195673658059"],[1589614934,"55.69594336539635"],[1589621414,"55.97313163686761"],[1589627894,"56.786190984293974"],[1589634374,"56.817295918166465"],[1589640854,"61.338142129861126"],[1589647334,"61.527921599817304"],[1589653814,"61.64486040184425"],[1589660294,"61.77833600415782"],[1589666774,"61.821646671575216"],[1589673254,"61.968902940794315"],[1589679734,"62.21931734513484"],[1589686214,"62.27365254607664"],[1589692694,"62.38822894806264"],[1589699174,"55.683343898511296"],[1589705654,"56.56688151382595"],[1589712134,"56.67437071568909"],[1589718614,"56.70547564956158"],[1589725094,"64.42422405001989"],[1589731574,"64.55297485225157"],[1589738054,"64.66558258753676"],[1589744534,"64.83921899054647"],[1589751014,"64.98883765980652"],[1589757494,"65.08608979482555"],[1589763974,"65.423519267341"],[1589770454,"65.5924308702688"],[1589776934,"65.77906047350372"],[1589783414,"63.95331897519087"],[1589789894,"56.08967670555439"],[1589796374,"56.26410057524441"],[1589802854,"56.41726284456588"],[1589809334,"63.49343843388626"],[1589815814,"63.809606306033174"],[1589822294,"61.586981600841014"],[1589828774,"61.63737946838125"],[1589835254,"61.97914000763842"],[1589841734,"62.15789494407018"],[1589848214,"62.45083254914776"],[1589854694,"62.653211485989004"],[1589861174,"62.82251682225696"],[1589867654,"62.358698947550785"],[1589874134,"55.99084963717472"],[1589880614,"56.36843991038629"],[1589887094,"56.49640324593766"],[1589893574,"56.61097964792364"],[1589900054,"64.06907657719732"],[1589906534,"64.38918178274582"],[1589913014,"64.50178951803102"],[1589919494,"64.4053248496923"],[1589925974,"58.69304155067939"],[1589932454,"58.994247555900294"],[1589938934,"59.14819729190208"],[1589945414,"59.275373160773135"],[1589951894,"59.25489902708492"],[1589958374,"55.760121899842105"],[1589964854,"56.091645372255186"],[1589971334,"56.26961284200662"],[1589977814,"56.78304111757271"],[1589984294,"62.006701341449485"],[1589990774,"62.4720941495163"],[1589997254,"62.69061615330401"],[1590003734,"62.80558628863017"],[1590010214,"62.31026974671134"],[1590016694,"62.344918280645246"],[1590023174,"62.51304241689274"],[1590029654,"62.64691175254647"],[1590036134,"62.794168021765586"],[1590042614,"58.300489410541815"],[1590049094,"56.32158564290748"],[1590055574,"56.41175057780367"],[1590062054,"56.42907484477063"],[1590068534,"66.87127675910213"],[1590075014,"63.41547923253498"],[1590081494,"63.612345902613995"],[1590087974,"63.79070710570559"],[1590094454,"63.9848176424035"],[1590100934,"64.13758617838484"],[1590107414,"64.44784805042937"],[1590113894,"64.66794498757771"],[1590120374,"64.92544659204107"],[1590126854,"65.04041672736722"],[1590133334,"55.870760968426524"],[1590139814,"56.92006031994771"],[1590146294,"57.03778658865497"],[1590152774,"56.927147520070555"],[1590159254,"65.77591060678245"],[1590165734,"66.05349261159387"],[1590172214,"66.13184554628533"],[1590178694,"65.58101260340422"],[1590185174,"65.7077947389351"],[1590191654,"65.77039834002024"],[1590198134,"65.81843380751951"],[1590204614,"65.89678674221096"],[1590211094,"65.86450060831801"],[1590217574,"55.855799101500516"],[1590224054,"56.03888510467401"],[1590230534,"56.249532441658566"],[1590237014,"56.449942711799004"],[1590243494,"65.71094460565638"],[1590249974,"65.75543647309424"],[1590256454,"65.91135487579682"],[1590262934,"60.49279664854181"],[1590269414,"60.8298323877171"],[1590275894,"60.95976438996925"],[1590282374,"61.06764732517256"],[1590288854,"61.4416939983227"],[1590295334,"61.58343800077959"],[1590301814,"58.251272743022064"],[1590308294,"55.515613495603965"],[1590314774,"56.27040030868694"],[1590321254,"56.40426964434068"],[1590327734,"65.59597447033022"],[1590334214,"66.10349674579395"],[1590340694,"65.84875127471169"],[1590347174,"66.1133400792979"],[1590353654,"66.26886474866033"],[1590360134,"66.2728020820619"]]}]}} + + + {"status":"success","data":{"resultType":"matrix","result":[{"metric":{},"values":[[1587768134,"51.542056626728986"],[1587774614,"52.020442635021006"],[1587781094,"52.37834624122467"],[1587787574,"52.76066131451813"],[1587794054,"53.076435453324876"],[1587800534,"44.07096649723009"],[1587807014,"44.70212104150343"],[1587813494,"45.22263651719236"],[1587819974,"45.74512065958209"],[1587826454,"53.23471625606842"],[1587832934,"53.66861039692258"],[1587839414,"53.965879068741906"],[1587845894,"54.20763133959894"],[1587852374,"52.61891731206123"],[1587858854,"53.1362829210289"],[1587865334,"53.43512652620886"],[1587871814,"53.82925359970706"],[1587878294,"54.01745813630261"],[1587884774,"46.972387480854714"],[1587891254,"47.647246425885605"],[1587897734,"47.91813496391434"],[1587904214,"48.1783927017588"],[1587910694,"56.29796164249801"],[1587917174,"55.89674736887695"],[1587923654,"56.29402430909642"],[1587930134,"56.56136924706373"],[1587936614,"56.62791018155045"],[1587943094,"56.18732257391359"],[1587949574,"56.39285137747609"],[1587956054,"55.85540536816036"],[1587962534,"56.63735978171424"],[1587969014,"52.26101370585757"],[1587975494,"49.91672539855657"],[1587981974,"50.097842735029275"],[1587988454,"51.35818315687517"],[1587994934,"57.60869993188413"],[1588001414,"57.74453793423866"],[1588007894,"58.00203953870202"],[1588014374,"58.25639127644412"],[1588020854,"58.43239007949476"],[1588027334,"58.66863008358959"],[1588033814,"59.46042783064741"],[1588040294,"59.706117434906034"],[1588046774,"59.742340902200574"],[1588053254,"59.74745943562263"],[1588059734,"51.33455915646569"],[1588066214,"51.8243634316223"],[1588072694,"52.80594064863631"],[1588079174,"52.968552518121584"],[1588085654,"60.73258025269806"],[1588092134,"60.15930450942795"],[1588098614,"60.27978691151631"],[1588105094,"60.449092247784264"],[1588111574,"60.51090838218908"],[1588118054,"60.247894510963505"],[1588124534,"56.15936750676237"],[1588131014,"56.37828324389025"],[1588137494,"55.933758302851814"],[1588143974,"51.0483150181708"],[1588150454,"51.368813957059444"],[1588156934,"51.647577161891334"],[1588163414,"52.327554640344275"],[1588169894,"58.49971848066179"],[1588176374,"58.60130168242256"],[1588182854,"59.11472995798866"],[1588189334,"59.69273050134066"],[1588195814,"59.70218010150445"],[1588202294,"59.61083396658778"],[1588208774,"60.3727079797936"],[1588215254,"60.589261316880524"],[1588221734,"60.67824505175625"],[1588228214,"57.0074691214628"],[1588234694,"52.60553037849586"],[1588241174,"52.750030514333865"],[1588247654,"52.844920249311954"],[1588254134,"59.189539292618676"],[1588260614,"59.525393831773485"],[1588267094,"59.73249756869663"],[1588273574,"59.887234771378736"],[1588280054,"60.09000744156013"],[1588286534,"60.28529917827852"],[1588293014,"55.68767496525303"],[1588299494,"56.61531071466539"],[1588305974,"56.97754538761078"],[1588312454,"57.08070352273219"],[1588318934,"52.54292677741073"],[1588325414,"52.75042424767402"],[1588331894,"52.92484811736403"],[1588338374,"53.02997491918623"],[1588344854,"65.66724020489883"],[1588351334,"65.79835340717146"],[1588357814,"65.3577657995346"],[1588364294,"59.78683276963843"],[1588370774,"60.39200091346135"],[1588377254,"60.567212249831684"],[1588383734,"61.894487339504444"],[1588390214,"62.03308147524008"],[1588396694,"61.97559640757701"],[1588403174,"53.87610786718587"],[1588409654,"54.12219120478464"],[1588416134,"54.370243209084215"],[1588422614,"54.50214387803716"],[1588429094,"62.339406013883035"],[1588435574,"59.50255729804432"],[1588442054,"59.81360663676918"],[1588448534,"59.98488063973794"],[1588455014,"60.113631441969616"],[1588461494,"60.255375444426505"],[1588467974,"60.48649691509927"],[1588474454,"60.56524358313089"],[1588480934,"60.67470145169482"],[1588487414,"57.417345528567324"],[1588493894,"54.637981880391685"],[1588500374,"54.83287988376991"],[1588506854,"55.1990518901169"],[1588513334,"62.47721268293836"],[1588519814,"63.006784025450926"],[1588526294,"63.28475976360251"],[1588532774,"63.351300698089204"],[1588539254,"63.70211710417003"],[1588545734,"63.82417443961902"],[1588552214,"63.90961457443331"],[1588558694,"64.04624004346816"],[1588565174,"64.16239137881479"],[1588571654,"64.24546911358813"],[1588578134,"55.72114229916647"],[1588584614,"55.98888097047394"],[1588591094,"56.145980573196994"],[1588597574,"56.13692470637336"],[1588604054,"62.8953574901862"],[1588610534,"62.59178908492434"],[1588617014,"62.746132554266296"],[1588623494,"62.88236428996098"],[1588629974,"63.34027616456479"],[1588636454,"63.80685017265208"],[1588642934,"64.1312864449423"],[1588649414,"64.40217498297103"],[1588655894,"64.66479512085644"],[1588662374,"56.281031108871204"],[1588668854,"56.79367191775698"],[1588675334,"56.839738718555466"],[1588681814,"57.234259525393824"],[1588688294,"65.77433567342182"],[1588694774,"65.76803593997928"],[1588701254,"65.82079620756048"],[1588707734,"66.50510475275514"],[1588714214,"66.48069328566535"],[1588720694,"66.31020674937692"],[1588727174,"64.84236885726773"],[1588733654,"64.64707712054934"],[1588740134,"64.82228845691967"],[1588746614,"62.07875454269841"],[1588753094,"55.63215856429075"],[1588759574,"55.6955496320562"],[1588766054,"55.75775949980116"],[1588772534,"63.28987829702456"],[1588779014,"63.52729950113986"],[1588785494,"63.93441977486327"],[1588791974,"64.12774284488087"],[1588798454,"64.70534965489273"],[1588804934,"57.931561270813724"],[1588811414,"57.955579004563376"],[1588817894,"58.00715807212408"],[1588824374,"58.12173447411006"],[1588830854,"58.300489410541815"],[1588837334,"55.20259549017832"],[1588843814,"56.10227617243946"],[1588850294,"56.15661137338126"],[1588856774,"56.22393977454829"],[1588863254,"63.04891349284784"],[1588869734,"64.34350871528748"],[1588876214,"63.96040617531371"],[1588882694,"64.19192137932663"],[1588889174,"64.09309431094697"],[1588895654,"64.25688738045271"],[1588902134,"63.71944137113699"],[1588908614,"63.92497017469948"],[1588915094,"64.12104937809819"],[1588921574,"55.9479327030975"],[1588928054,"55.649876564597854"],[1588934534,"55.750672299678314"],[1588941014,"55.86367376830368"],[1588947494,"62.53824135066285"],[1588953974,"62.677622953078796"],[1588960454,"62.461463349332035"],[1588966934,"63.046157359466726"],[1588973414,"63.60525870249115"],[1588979894,"63.746608971607884"],[1588986374,"64.45336031719158"],[1588992854,"64.54549391878857"],[1588999334,"63.999385775989346"],[1589005814,"59.296241027801514"],[1589012294,"56.45033644513916"],[1589018774,"56.51963351300699"],[1589025254,"56.64680938187803"],[1589031734,"64.08403844412334"],[1589038214,"64.1950712460479"],[1589044694,"64.24861898030939"],[1589051174,"60.40145051362514"],[1589057654,"60.43491784753857"],[1589064134,"60.52941384917651"],[1589070614,"60.711318652329524"],[1589077094,"60.75620425310754"],[1589083574,"60.88731745538018"],[1589090054,"60.956220789907825"],[1589096534,"56.18653510723327"],[1589103014,"56.94801538709894"],[1589109494,"57.14094472377638"],[1589115974,"57.29883179317975"],[1589122454,"64.34468991530797"],[1589128934,"64.43367365018368"],[1589135414,"64.57226778591931"],[1589141894,"64.80023938987081"],[1589148374,"64.9758444595813"],[1589154854,"65.16523019619733"],[1589161334,"65.20657219691391"],[1589167814,"65.38847700006694"],[1589174294,"65.46171140133633"],[1589180774,"55.98888097047394"],[1589187254,"56.30071777587911"],[1589193734,"56.54798231349836"],[1589200214,"57.264576992586"],[1589206694,"63.6985735041086"],[1589213174,"63.1619149614732"],[1589219654,"63.45327763319014"],[1589226134,"63.68558030388339"],[1589232614,"63.87063497375768"],[1589239094,"64.05096484355006"],[1589245574,"65.18412939652491"],[1589252054,"64.95852019261436"],[1589258534,"64.82898192370236"],[1589265014,"60.18765330991932"],[1589271494,"56.716500183086005"],[1589277974,"56.7306745833317"],[1589284454,"56.94959032045956"],[1589290934,"64.22853857996134"],[1589297414,"59.61634623335"],[1589303894,"60.08370770811759"],[1589310374,"60.19316557668154"],[1589316854,"58.702884884183334"],[1589323334,"59.011571822867246"],[1589329814,"59.25962382716682"],[1589336294,"59.39821796290244"],[1589342774,"59.480508230995476"],[1589349254,"59.56279849908851"],[1589355734,"55.39316242681481"],[1589362214,"55.45379736119915"],[1589368694,"55.59829749703715"],[1589375174,"55.74122269951452"],[1589381654,"63.815512306135545"],[1589388134,"64.00332310939093"],[1589394614,"58.54893514818155"],[1589401094,"59.167096492229675"],[1589407574,"59.240724626839224"],[1589414054,"59.47617716425374"],[1589420534,"60.130168242256254"],[1589427014,"59.877391437874785"],[1589433494,"59.98094330633636"],[1589439974,"56.420018977946995"],[1589446454,"56.794065651097135"],[1589452934,"56.97557672090999"],[1589459414,"56.931084853472136"],[1589465894,"58.31702621082846"],[1589472374,"58.846203820000866"],[1589478854,"58.95408675520417"],[1589485334,"59.17221502565173"],[1589491814,"59.3714440957717"],[1589498294,"59.584060099457034"],[1589504774,"59.792345036400654"],[1589511254,"59.908102638407115"],[1589517734,"60.09512597498219"],[1589524214,"59.02732115647357"],[1589530694,"56.607829781202376"],[1589537174,"56.6995696494592"],[1589543654,"56.83501391847358"],[1589550134,"63.209556695632315"],[1589556614,"63.340669897904945"],[1589563094,"63.504069234070535"],[1589569574,"63.65683777005186"],[1589576054,"63.94820044176881"],[1589582534,"63.99505470924761"],[1589589014,"64.5423440520673"],[1589595494,"64.98371912638447"],[1589601974,"65.10695766185394"],[1589608454,"65.57195673658059"],[1589614934,"55.69594336539635"],[1589621414,"55.97313163686761"],[1589627894,"56.786190984293974"],[1589634374,"56.817295918166465"],[1589640854,"61.338142129861126"],[1589647334,"61.527921599817304"],[1589653814,"61.64486040184425"],[1589660294,"61.77833600415782"],[1589666774,"61.821646671575216"],[1589673254,"61.968902940794315"],[1589679734,"62.21931734513484"],[1589686214,"62.27365254607664"],[1589692694,"62.38822894806264"],[1589699174,"55.683343898511296"],[1589705654,"56.56688151382595"],[1589712134,"56.67437071568909"],[1589718614,"56.70547564956158"],[1589725094,"64.42422405001989"],[1589731574,"64.55297485225157"],[1589738054,"64.66558258753676"],[1589744534,"64.83921899054647"],[1589751014,"64.98883765980652"],[1589757494,"65.08608979482555"],[1589763974,"65.423519267341"],[1589770454,"65.5924308702688"],[1589776934,"65.77906047350372"],[1589783414,"63.95331897519087"],[1589789894,"56.08967670555439"],[1589796374,"56.26410057524441"],[1589802854,"56.41726284456588"],[1589809334,"63.49343843388626"],[1589815814,"63.809606306033174"],[1589822294,"61.586981600841014"],[1589828774,"61.63737946838125"],[1589835254,"61.97914000763842"],[1589841734,"62.15789494407018"],[1589848214,"62.45083254914776"],[1589854694,"62.653211485989004"],[1589861174,"62.82251682225696"],[1589867654,"62.358698947550785"],[1589874134,"55.99084963717472"],[1589880614,"56.36843991038629"],[1589887094,"56.49640324593766"],[1589893574,"56.61097964792364"],[1589900054,"64.06907657719732"],[1589906534,"64.38918178274582"],[1589913014,"64.50178951803102"],[1589919494,"64.4053248496923"],[1589925974,"58.69304155067939"],[1589932454,"58.994247555900294"],[1589938934,"59.14819729190208"],[1589945414,"59.275373160773135"],[1589951894,"59.25489902708492"],[1589958374,"55.760121899842105"],[1589964854,"56.091645372255186"],[1589971334,"56.26961284200662"],[1589977814,"56.78304111757271"],[1589984294,"62.006701341449485"],[1589990774,"62.4720941495163"],[1589997254,"62.69061615330401"],[1590003734,"62.80558628863017"],[1590010214,"62.31026974671134"],[1590016694,"62.344918280645246"],[1590023174,"62.51304241689274"],[1590029654,"62.64691175254647"],[1590036134,"62.794168021765586"],[1590042614,"58.300489410541815"],[1590049094,"56.32158564290748"],[1590055574,"56.41175057780367"],[1590062054,"56.42907484477063"],[1590068534,"66.87127675910213"],[1590075014,"63.41547923253498"],[1590081494,"63.612345902613995"],[1590087974,"63.79070710570559"],[1590094454,"63.9848176424035"],[1590100934,"64.13758617838484"],[1590107414,"64.44784805042937"],[1590113894,"64.66794498757771"],[1590120374,"64.92544659204107"],[1590126854,"65.04041672736722"],[1590133334,"55.870760968426524"],[1590139814,"56.92006031994771"],[1590146294,"57.03778658865497"],[1590152774,"56.927147520070555"],[1590159254,"65.77591060678245"],[1590165734,"66.05349261159387"],[1590172214,"66.13184554628533"],[1590178694,"65.58101260340422"],[1590185174,"65.7077947389351"],[1590191654,"65.77039834002024"],[1590198134,"65.81843380751951"],[1590204614,"65.89678674221096"],[1590211094,"65.86450060831801"],[1590217574,"55.855799101500516"],[1590224054,"56.03888510467401"],[1590230534,"56.249532441658566"],[1590237014,"56.449942711799004"],[1590243494,"65.71094460565638"],[1590249974,"65.75543647309424"],[1590256454,"65.91135487579682"],[1590262934,"60.49279664854181"],[1590269414,"60.8298323877171"],[1590275894,"60.95976438996925"],[1590282374,"61.06764732517256"],[1590288854,"61.4416939983227"],[1590295334,"61.58343800077959"],[1590301814,"58.251272743022064"],[1590308294,"55.515613495603965"],[1590314774,"56.27040030868694"],[1590321254,"56.40426964434068"],[1590327734,"65.59597447033022"],[1590334214,"66.10349674579395"],[1590340694,"65.84875127471169"],[1590347174,"66.1133400792979"],[1590353654,"66.26886474866033"],[1590360134,"66.2728020820619"]]}]}} + + */ -// //This will get triggered roughly every minute (10 seconds in debug), but we don't want to healthcheck that frequently -// if (!DateUtil.IsAfterDuration(lastServerCheckDone, DO_EVERY_INTERVAL)) -// return; + //This will get triggered roughly every minute (10 seconds in debug), but we don't want to healthcheck that frequently + if (!DateUtil.IsAfterDuration(lastServerCheckDone, DO_EVERY_INTERVAL)) + return; -// log.LogTrace("Starting metrics snapshot"); + log.LogTrace("Starting metrics snapshot"); -// //Gather core metrics here -// var process = Process.GetCurrentProcess(); + //Gather core metrics here + var process = Process.GetCurrentProcess(); -// //PHYSICAL MEMORY -// metrics.Measure.Gauge.SetValue(MetricsRegistry.PhysicalMemoryGauge, process.WorkingSet64); + //PHYSICAL MEMORY + // metrics.Measure.Gauge.SetValue(MetricsRegistry.PhysicalMemoryGauge, process.WorkingSet64); -// //PRIVATE BYTES -// metrics.Measure.Gauge.SetValue(MetricsRegistry.PrivateBytesGauge, process.PrivateMemorySize64); + //PRIVATE BYTES + // metrics.Measure.Gauge.SetValue(MetricsRegistry.PrivateBytesGauge, process.PrivateMemorySize64); -// //RECORDS IN TABLE -// //Only do this once per hour -// if (DateUtil.IsAfterDuration(lastRecordCountCheck, 1)) -// { -// lastRecordCountCheck = DateTime.UtcNow; -// log.LogTrace("Counting table records"); + //RECORDS IN TABLE + //Only do this once per hour + if (DateUtil.IsAfterDuration(lastRecordCountCheck, 1)) + { + lastRecordCountCheck = DateTime.UtcNow; + log.LogTrace("Counting table records"); -// //Get a count of important tables in db -// List allTableNames = await DbUtil.GetAllTablenamesAsync(); + //Get a count of important tables in db + List allTableNames = await DbUtil.GetAllTablenamesAsync(); -// //Skip some tables as they are internal and / or only ever have one record -// List skipTableNames = new List(); -// skipTableNames.Add("alicense"); -// skipTableNames.Add("aschemaversion"); + //Skip some tables as they are internal and / or only ever have one record + List skipTableNames = new List(); + skipTableNames.Add("alicense"); + skipTableNames.Add("aschemaversion"); -// foreach (string table in allTableNames) -// { -// if (!skipTableNames.Contains(table)) -// { -// var tags = new MetricTags("TableTagKey", table); -// metrics.Measure.Gauge.SetValue(MetricsRegistry.DBRecordsGauge, tags, await DbUtil.CountOfRecordsAsync(table)); -// } -// } -// } + foreach (string table in allTableNames) + { + if (!skipTableNames.Contains(table)) + { + //var tags = new MetricTags("TableTagKey", table); + // metrics.Measure.Gauge.SetValue(MetricsRegistry.DBRecordsGauge, tags, await DbUtil.CountOfRecordsAsync(table)); + } + } + } -// //JOB COUNTS (DEAD, RUNNING, COMPLETED, SLEEPING) + //JOB COUNTS (DEAD, RUNNING, COMPLETED, SLEEPING) -// foreach (JobStatus stat in Enum.GetValues(typeof(JobStatus))) -// { -// var jobtag = new MetricTags("JobStatus", stat.ToString()); -// metrics.Measure.Gauge.SetValue(MetricsRegistry.JobsGauge, jobtag, await JobsBiz.GetCountForJobStatusAsync(ct, stat)); -// } + foreach (JobStatus stat in Enum.GetValues(typeof(JobStatus))) + { + // var jobtag = new MetricTags("JobStatus", stat.ToString()); + // metrics.Measure.Gauge.SetValue(MetricsRegistry.JobsGauge, jobtag, await JobsBiz.GetCountForJobStatusAsync(ct, stat)); + } -// //FILES ON DISK -// //Only do this once per hour -// if (DateUtil.IsAfterDuration(lastFileCountCheck, 1)) -// { -// lastFileCountCheck = DateTime.UtcNow; -// log.LogTrace("Files on disk information"); -// var UtilFilesInfo = FileUtil.GetUtilityFolderSizeInfo(); -// var UserFilesInfo = FileUtil.GetAttachmentFolderSizeInfo(); + //FILES ON DISK + //Only do this once per hour + if (DateUtil.IsAfterDuration(lastFileCountCheck, 1)) + { + lastFileCountCheck = DateTime.UtcNow; + log.LogTrace("Files on disk information"); + var UtilFilesInfo = FileUtil.GetUtilityFolderSizeInfo(); + var UserFilesInfo = FileUtil.GetAttachmentFolderSizeInfo(); -// var mtag = new MetricTags("File type", "Business object files"); -// metrics.Measure.Gauge.SetValue(MetricsRegistry.FileCountGauge, mtag, UserFilesInfo.FileCountWithChildren); -// metrics.Measure.Gauge.SetValue(MetricsRegistry.FileSizeGauge, mtag, UserFilesInfo.SizeWithChildren); + // var mtag = new MetricTags("File type", "Business object files"); + // metrics.Measure.Gauge.SetValue(MetricsRegistry.FileCountGauge, mtag, UserFilesInfo.FileCountWithChildren); + // metrics.Measure.Gauge.SetValue(MetricsRegistry.FileSizeGauge, mtag, UserFilesInfo.SizeWithChildren); -// mtag = new MetricTags("File type", "OPS files"); -// metrics.Measure.Gauge.SetValue(MetricsRegistry.FileCountGauge, mtag, UtilFilesInfo.FileCountWithChildren); -// metrics.Measure.Gauge.SetValue(MetricsRegistry.FileSizeGauge, mtag, UtilFilesInfo.SizeWithChildren); + // mtag = new MetricTags("File type", "OPS files"); + // metrics.Measure.Gauge.SetValue(MetricsRegistry.FileCountGauge, mtag, UtilFilesInfo.FileCountWithChildren); + // metrics.Measure.Gauge.SetValue(MetricsRegistry.FileSizeGauge, mtag, UtilFilesInfo.SizeWithChildren); -// } + } -// lastServerCheckDone = DateTime.UtcNow; + lastServerCheckDone = DateTime.UtcNow; -// //just to hide compiler warning for now -// await Task.CompletedTask; + //just to hide compiler warning for now + await Task.CompletedTask; -// } + } -// ///////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////// -// }//eoc + }//eoc -// }//eons +}//eons