diff --git a/devdocs/todo.txt b/devdocs/todo.txt index bafe6380..a3e7eb8e 100644 --- a/devdocs/todo.txt +++ b/devdocs/todo.txt @@ -38,525 +38,6 @@ Lessons learned: -WIP >>>>>>>>>>>>>>>>> - -Try to find the common thign to look for when db is down: - -//Inner exception -Npgsql.NpgsqlException (0x80004005): Failed to connect to 127.0.0.1:5432 - -System.Net.Sockets.SocketException (10054): An existing connection was forcibly closed by the remote host. -System.Net.Sockets.SocketException (10061): No connection could be made because the target machine actively refused it. - -ex.source "Npgsql.EntityFrameworkCore.PostgreSQL","Npgsql" -ex.message "An exception has been raised that is likely due to a transient failure.","Exception while writing to stream","Unable to write data to the transport connection: An existing connection was forcibly closed by the remote host.." - - -Boot exception when db is not up: -{Npgsql.NpgsqlException (0x80004005): Failed to connect to 127.0.0.1:5432 - ---> System.Net.Sockets.SocketException (10061): No connection could be made because the target machine actively refused it. - at Npgsql.Internal.NpgsqlConnector.Connect(NpgsqlTimeout timeout) - at Npgsql.Internal.NpgsqlConnector.Connect(NpgsqlTimeout timeout) - at Npgsql.Internal.NpgsqlConnector.RawOpen(SslMode sslMode, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken, Boolean isFirstAttempt) - at Npgsql.Internal.NpgsqlConnector.g__OpenCore|191_1(NpgsqlConnector conn, SslMode sslMode, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken, Boolean isFirstAttempt) - at Npgsql.Internal.NpgsqlConnector.Open(NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) - at Npgsql.ConnectorPool.OpenNewConnector(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) - at Npgsql.ConnectorPool.g__RentAsync|28_0(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) - at Npgsql.NpgsqlConnection.g__OpenAsync|45_0(Boolean async, CancellationToken cancellationToken) - at Npgsql.NpgsqlConnection.Open() - at AyaNova.Util.DbUtil.<>c.b__14_0() in C:\data\code\raven\server\AyaNova\util\DbUtil.cs:line 195} - - -When running: -{System.InvalidOperationException: An exception has been raised that is likely due to a transient failure. - ---> Npgsql.NpgsqlException (0x80004005): Failed to connect to 127.0.0.1:5432 - ---> System.Net.Sockets.SocketException (10061): No connection could be made because the target machine actively refused it. - at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.ThrowException(SocketError error, CancellationToken cancellationToken) - at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.System.Threading.Tasks.Sources.IValueTaskSource.GetResult(Int16 token) - at System.Net.Sockets.Socket.g__WaitForConnectWithCancellation|277_0(AwaitableSocketAsyncEventArgs saea, ValueTask connectTask, CancellationToken cancellationToken) - at Npgsql.TaskExtensions.ExecuteWithTimeout(Func`2 func, NpgsqlTimeout timeout, CancellationToken cancellationToken) - at Npgsql.Internal.NpgsqlConnector.ConnectAsync(NpgsqlTimeout timeout, CancellationToken cancellationToken) - at Npgsql.Internal.NpgsqlConnector.ConnectAsync(NpgsqlTimeout timeout, CancellationToken cancellationToken) - at Npgsql.Internal.NpgsqlConnector.RawOpen(SslMode sslMode, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken, Boolean isFirstAttempt) - at Npgsql.Internal.NpgsqlConnector.g__OpenCore|191_1(NpgsqlConnector conn, SslMode sslMode, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken, Boolean isFirstAttempt) - at Npgsql.Internal.NpgsqlConnector.Open(NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) - at Npgsql.ConnectorPool.OpenNewConnector(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) - at Npgsql.ConnectorPool.g__RentAsync|28_0(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) - at Npgsql.NpgsqlConnection.g__OpenAsync|45_0(Boolean async, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenInternalAsync(Boolean errorsExpected, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenInternalAsync(Boolean errorsExpected, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenAsync(CancellationToken cancellationToken, Boolean errorsExpected) - at Microsoft.EntityFrameworkCore.Storage.RelationalCommand.ExecuteReaderAsync(RelationalCommandParameterObject parameterObject, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.InitializeReaderAsync(AsyncEnumerator enumerator, CancellationToken cancellationToken) - at Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.NpgsqlExecutionStrategy.ExecuteAsync[TState,TResult](TState state, Func`4 operation, Func`4 verifySucceeded, CancellationToken cancellationToken) - --- End of inner exception stack trace --- - at Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.NpgsqlExecutionStrategy.ExecuteAsync[TState,TResult](TState state, Func`4 operation, Func`4 verifySucceeded, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.MoveNextAsync() - at Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions.ToListAsync[TSource](IQueryable`1 source, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions.ToListAsync[TSource](IQueryable`1 source, CancellationToken cancellationToken) - at AyaNova.Biz.NotifyEventHelper.AddGeneralNotifyEvent(AyaType atype, Int64 objectid, NotifyEventType eventType, String message, String name, Exception except, Int64 userId) in C:\data\code\raven\server\AyaNova\biz\NotifyEventHelper.cs:line 376} - -<<<<<<<<<<<<<<<<<<< /WIP` - - - - -COMMANDS USED: -show log file usage of system logs -root@devops:/var/ayanova/data/logs# du -h /var/log/ -4.0K /var/log/private -8.0K /var/log/postgresql -8.1M /var/log/journal/c3a57680c1d26ca313b9c7ec36a5beaa -2.4G /var/log/journal/d5795bc80e579af45b03fda86314e673 -2.4G /var/log/journal -4.0K /var/log/sysstat -32K /var/log/unattended-upgrades -4.0K /var/log/landscape -136K /var/log/letsencrypt -156K /var/log/nginx -4.0K /var/log/dist-upgrade -160K /var/log/apt -11G /var/log/ - - - -In var log is this: --rw-r----- 1 syslog adm 8.3G Sep 10 10:17 syslog - - - -Find the biggest folders from root directory: -root@devops:/# du -h --max-depth=1 | sort -rh -du: cannot access './proc/36921/task/36921/fd/4': No such file or directory -du: cannot access './proc/36921/task/36921/fdinfo/4': No such file or directory -du: cannot access './proc/36921/fd/3': No such file or directory -du: cannot access './proc/36921/fdinfo/3': No such file or directory -8.9G . <- total used -2.8G ./srv -2.3G ./usr -1.5G ./var -1.2G ./snap -110M ./boot -17M ./opt -6.6M ./etc -1012K ./run -144K ./root -72K ./tmp -16K ./lost+found -4.0K ./mnt -4.0K ./media -4.0K ./home -0 ./sys -0 ./proc -0 ./dev - -Then go into each folder and run again to drill down and find it - -So on devops I'm about right at this point once I removed the log files from system journal and ayanova - -POSTGRES VIEW STATUS STOP / START -root@devops:/var/ayanova# service postgresql status -○ postgresql.service - PostgreSQL RDBMS - Loaded: loaded (/lib/systemd/system/postgresql.service; enabled; vendor preset: enabled) - Active: inactive (dead) since Thu 2022-09-08 15:36:20 PDT; 1 day 19h ago - Main PID: 4401 (code=exited, status=0/SUCCESS) - CPU: 1ms - -Notice: journal has been rotated since unit was started, output may be incomplete. - - - - -cap ayanova log file size or something I just found a 1gb log file in devops - NOTE it's logging in ayanova log and also in linux system log journalctl log file location wherever that is so it's fucking up multiple things - I guess exceptions log automatically as output in command line of server via system service as well as ayanvoa's log - So, can I turn off logging of the service output or limit it as well as separately the ayanvoa log - - if it can't connect to the db it just fills gb of log files with this: - Sep 10 08:45:01 devops CRON[36636]: pam_unix(cron:session): session opened for user root(uid=0) by (uid=0) -Sep 10 08:44:39 devops systemd[1]: ayanova.service: Consumed 1h 53min 39.035s CPU time. -Sep 10 08:44:39 devops systemd[1]: Stopped AyaNova server. -Sep 10 08:44:39 devops systemd[1]: ayanova.service: Deactivated successfully. -Sep 10 08:44:39 devops systemd[1]: Stopping AyaNova server... -Sep 10 08:44:38 devops ayanova-server[6549]: at AyaNova.Biz.NotifyEventHelper.AddGeneralNotifyEvent(AyaType atype, Int64 objectid, NotifyEventType eventType, String message, String name, Exception except, Int64 userId) in C:\data\code\raven\server\Ay> -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions.ToListAsync[TSource](IQueryable`1 source, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions.ToListAsync[TSource](IQueryable`1 source, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.MoveNextAsync() -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.NpgsqlExecutionStrategy.ExecuteAsync[TState,TResult](TState state, Func`4 operation, Func`4 verifySucceeded, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: --- End of inner exception stack trace --- -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.NpgsqlExecutionStrategy.ExecuteAsync[TState,TResult](TState state, Func`4 operation, Func`4 verifySucceeded, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.InitializeReaderAsync(AsyncEnumerator enumerator, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Storage.RelationalCommand.ExecuteReaderAsync(RelationalCommandParameterObject parameterObject, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenAsync(CancellationToken cancellationToken, Boolean errorsExpected) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenInternalAsync(Boolean errorsExpected, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenInternalAsync(Boolean errorsExpected, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.NpgsqlConnection.g__OpenAsync|45_0(Boolean async, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.ConnectorPool.g__RentAsync|28_0(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.ConnectorPool.OpenNewConnector(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.Internal.NpgsqlConnector.Open(NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.Internal.NpgsqlConnector.g__OpenCore|191_1(NpgsqlConnector conn, SslMode sslMode, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken, Boolean isFirstAttempt) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.Internal.NpgsqlConnector.RawOpen(SslMode sslMode, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken, Boolean isFirstAttempt) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.Internal.NpgsqlConnector.ConnectAsync(NpgsqlTimeout timeout, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.Internal.NpgsqlConnector.ConnectAsync(NpgsqlTimeout timeout, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.TaskExtensions.ExecuteWithTimeout(Func`2 func, NpgsqlTimeout timeout, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: --- End of stack trace from previous location --- -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart() -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Threading.ThreadPoolWorkQueue.Dispatch() -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Threading.UnmanagedThreadPoolWorkItem.System.Threading.IThreadPoolWorkItem.Execute() -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Threading.UnmanagedThreadPoolWorkItem.ExecuteUnmanagedThreadPoolWorkItem(IntPtr callback, IntPtr state) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Threading.UnmanagedThreadPoolWorkItem.ExecuteUnmanagedThreadPoolWorkItem(IntPtr callback, IntPtr state) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Threading.TimerQueue.FireNextTimers() -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Threading.TimerQueueTimer.Fire(Boolean isThreadPool) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Threading.Tasks.Task.DelayPromise.CompleteTimedOut() -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Threading.Tasks.Task.TrySetResult() -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Threading.Tasks.Task.RunContinuations(Object continuationObject) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Threading.Tasks.AwaitTaskContinuation.RunOrScheduleAction(IAsyncStateMachineBox box, Boolean allowInlining) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncTaskMethodBuilder`1.AsyncStateMachineBox`1.MoveNext(Thread threadPoolThread) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Threading.ExecutionContext.RunInternal(ExecutionContext executionContext, ContextCallback callback, Object state) -Sep 10 08:44:38 devops ayanova-server[6549]: at AyaNova.Generator.GeneratorService.ExecuteAsync(CancellationToken stoppingToken) in C:\data\code\raven\server\AyaNova\generator\Generate.cs:line 49 -Sep 10 08:44:38 devops ayanova-server[6549]: at AyaNova.Biz.JobsBiz.ProcessJobsAsync() -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 08:44:38 devops ayanova-server[6549]: at AyaNova.Biz.JobsBiz.ProcessJobsAsync() in C:\data\code\raven\server\AyaNova\biz\JobsBiz.cs:line 309 -lines 1-85 -Sep 10 08:44:38 devops ayanova-server[6549]: at AyaNova.Biz.NotifyEventHelper.AddOpsProblemEvent(String message, Exception ex) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 08:44:38 devops ayanova-server[6549]: at AyaNova.Biz.NotifyEventHelper.AddOpsProblemEvent(String message, Exception ex) -Sep 10 08:44:38 devops ayanova-server[6549]: at AyaNova.Biz.NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType eventType, String message, String name, Exception except, Int64 userId) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 08:44:38 devops ayanova-server[6549]: at AyaNova.Biz.NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType eventType, String message, String name, Exception except, Int64 userId) in C:\data\code\raven\server\AyaNova\biz\NotifyEventHelper.cs:> -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 08:44:38 devops ayanova-server[6549]: at AyaNova.Biz.NotifyEventHelper.AddGeneralNotifyEvent(AyaType atype, Int64 objectid, NotifyEventType eventType, String message, String name, Exception except, Int64 userId) in C:\data\code\raven\server\Ay> -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions.ToListAsync[TSource](IQueryable`1 source, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions.ToListAsync[TSource](IQueryable`1 source, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Runtime.CompilerServices.ConfiguredCancelableAsyncEnumerable`1.Enumerator.MoveNextAsync() -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.MoveNextAsync() -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.MoveNextAsync() -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.NpgsqlExecutionStrategy.ExecuteAsync[TState,TResult](TState state, Func`4 operation, Func`4 verifySucceeded, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.NpgsqlExecutionStrategy.ExecuteAsync[TState,TResult](TState state, Func`4 operation, Func`4 verifySucceeded, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.<>c.b__18_0(DbContext _, AsyncEnumerator enumerator, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.InitializeReaderAsync(AsyncEnumerator enumerator, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.InitializeReaderAsync(AsyncEnumerator enumerator, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Storage.RelationalCommand.ExecuteReaderAsync(RelationalCommandParameterObject parameterObject, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Storage.RelationalCommand.ExecuteReaderAsync(RelationalCommandParameterObject parameterObject, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenAsync(CancellationToken cancellationToken, Boolean errorsExpected) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenAsync(CancellationToken cancellationToken, Boolean errorsExpected) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenInternalAsync(Boolean errorsExpected, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenInternalAsync(Boolean errorsExpected, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenDbConnectionAsync(Boolean errorsExpected, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.NpgsqlConnection.OpenAsync(CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.NpgsqlConnection.Open(Boolean async, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.NpgsqlConnection.g__OpenAsync|45_0(Boolean async, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.NpgsqlConnection.g__OpenAsync|45_0(Boolean async, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.ConnectorPool.Get(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.ConnectorPool.g__RentAsync|28_0(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.ConnectorPool.g__RentAsync|28_0(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.ConnectorPool.OpenNewConnector(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.ConnectorPool.OpenNewConnector(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.Internal.NpgsqlConnector.Open(NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.Internal.NpgsqlConnector.Open(NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.Internal.NpgsqlConnector.g__OpenCore|191_1(NpgsqlConnector conn, SslMode sslMode, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken, Boolean isFirstAttempt) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.Internal.NpgsqlConnector.RawOpen(SslMode sslMode, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken, Boolean isFirstAttempt) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.Internal.NpgsqlConnector.RawOpen(SslMode sslMode, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken, Boolean isFirstAttempt) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.Internal.NpgsqlConnector.ConnectAsync(NpgsqlTimeout timeout, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.Internal.NpgsqlConnector.ConnectAsync(NpgsqlTimeout timeout, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.TaskExtensions.ExecuteWithTimeout(Func`2 func, NpgsqlTimeout timeout, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.TaskExtensions.ExecuteWithTimeout(Func`2 func, NpgsqlTimeout timeout, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.Internal.NpgsqlConnector.<>c__DisplayClass198_0.b__4(CancellationToken ct) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Net.Sockets.Socket.ConnectAsync(EndPoint remoteEP, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.ConnectAsync(Socket socket) -Sep 10 08:44:38 devops ayanova-server[6549]: at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.CreateException(SocketError error, Boolean forAsyncThrow) -Sep 10 08:44:38 devops ayanova-server[6549]: ---> System.Net.Sockets.SocketException (111): Connection refused -Sep 10 08:44:38 devops ayanova-server[6549]: ---> Npgsql.NpgsqlException (0x80004005): Failed to connect to 127.0.0.1:5432 -Sep 10 08:44:38 devops ayanova-server[6549]: ]|System.InvalidOperationException: An exception has been raised that is likely due to a transient failure. -Sep 10 08:44:38 devops ayanova-server[6549]: Npgsql -> Connection refused -Sep 10 08:44:38 devops ayanova-server[6549]: Npgsql -> Failed to connect to 127.0.0.1:5432 -Sep 10 08:44:38 devops ayanova-server[6549]: Exception error: Npgsql.EntityFrameworkCore.PostgreSQL -> An exception has been raised that is likely due to a transient failure. -Sep 10 08:44:38 devops ayanova-server[6549]: 2022-09-10 15:44:38.4500|ERROR|NotifyEventProcessor|Error adding general notify event [type:ServerOperationsProblem, userId:0, message:Server::ProcessJobsAsync unexpected error during processing -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.MoveNextAsync() -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.NpgsqlExecutionStrategy.ExecuteAsync[TState,TResult](TState state, Func`4 operation, Func`4 verifySucceeded, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: --- End of inner exception stack trace --- -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.NpgsqlExecutionStrategy.ExecuteAsync[TState,TResult](TState state, Func`4 operation, Func`4 verifySucceeded, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.InitializeReaderAsync(AsyncEnumerator enumerator, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Storage.RelationalCommand.ExecuteReaderAsync(RelationalCommandParameterObject parameterObject, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenAsync(CancellationToken cancellationToken, Boolean errorsExpected) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenInternalAsync(Boolean errorsExpected, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenInternalAsync(Boolean errorsExpected, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.NpgsqlConnection.g__OpenAsync|45_0(Boolean async, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.ConnectorPool.g__RentAsync|28_0(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.ConnectorPool.OpenNewConnector(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.Internal.NpgsqlConnector.Open(NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.Internal.NpgsqlConnector.g__OpenCore|191_1(NpgsqlConnector conn, SslMode sslMode, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken, Boolean isFirstAttempt) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.Internal.NpgsqlConnector.RawOpen(SslMode sslMode, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken, Boolean isFirstAttempt) -Sep 10 08:44:38 devops ayanova-server[6549]: at Npgsql.Internal.NpgsqlConnector.ConnectAsync(NpgsqlTimeout timeout, CancellationToken cancellationToken) -lines 86-170 - - - - -Sep 10 05:25:19 devops ayanova-server[6549]: at Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.NpgsqlExecutionStrategy.ExecuteAsync[TState,TResult](TState state, Func`4 operation, Func`4 verifySucceeded, CancellationToken cancellationToken) -Sep 10 05:25:19 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 05:25:19 devops ayanova-server[6549]: at Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.NpgsqlExecutionStrategy.ExecuteAsync[TState,TResult](TState state, Func`4 operation, Func`4 verifySucceeded, CancellationToken cancellationToken) -Sep 10 05:25:19 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.MoveNextAsync() -Sep 10 05:25:19 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 05:25:19 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.MoveNextAsync() -Sep 10 05:25:19 devops ayanova-server[6549]: at System.Runtime.CompilerServices.ConfiguredCancelableAsyncEnumerable`1.Enumerator.MoveNextAsync() -Sep 10 05:25:19 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions.ToListAsync[TSource](IQueryable`1 source, CancellationToken cancellationToken) -Sep 10 05:25:19 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 05:25:19 devops ayanova-server[6549]: at Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions.ToListAsync[TSource](IQueryable`1 source, CancellationToken cancellationToken) -Sep 10 05:25:19 devops ayanova-server[6549]: at AyaNova.Biz.NotifyEventHelper.AddGeneralNotifyEvent(AyaType atype, Int64 objectid, NotifyEventType eventType, String message, String name, Exception except, Int64 userId) in C:\data\code\raven\server\Ay> -Sep 10 05:25:19 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 05:25:19 devops ayanova-server[6549]: at AyaNova.Biz.NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType eventType, String message, String name, Exception except, Int64 userId) in C:\data\code\raven\server\AyaNova\biz\NotifyEventHelper.cs:> -Sep 10 05:25:19 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 05:25:19 devops ayanova-server[6549]: at AyaNova.Biz.NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType eventType, String message, String name, Exception except, Int64 userId) -Sep 10 05:25:19 devops ayanova-server[6549]: at AyaNova.Biz.NotifyEventHelper.AddOpsProblemEvent(String message, Exception ex) -Sep 10 05:25:19 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 05:25:19 devops ayanova-server[6549]: at AyaNova.Biz.NotifyEventHelper.AddOpsProblemEvent(String message, Exception ex) -Sep 10 05:25:19 devops ayanova-server[6549]: at AyaNova.Biz.JobsBiz.ProcessJobsAsync() in C:\data\code\raven\server\AyaNova\biz\JobsBiz.cs:line 309 -Sep 10 05:25:19 devops ayanova-server[6549]: at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) -Sep 10 05:25:19 devops ayanova-server[6549]: at AyaNova.Biz.JobsBiz.ProcessJobsAsync() -Sep 10 05:25:19 devops ayanova-server[6549]: at AyaNova.Generator.GeneratorService.ExecuteAsync(CancellationToken stoppingToken) in C:\data\code\raven\server\AyaNova\generator\Generate.cs:line 49 -Sep 10 05:25:19 devops ayanova-server[6549]: at System.Threading.ExecutionContext.RunInternal(ExecutionContext executionContext, ContextCallback callback, Object state) - - -Sep 10 05:26:41 devops ayanova-server[6549]: at AyaNova.Biz.CoreJobMetricsSnapshot.DoWork() in C:\data\code\raven\server\AyaNova\generator\CoreJobMetricsSnapshot.cs:line 180 -Sep 10 05:26:41 devops ayanova-server[6549]: at AyaNova.Biz.JobsBiz.ProcessJobsAsync() in C:\data\code\raven\server\AyaNova\biz\JobsBiz.cs:line 202 -Sep 10 05:26:41 devops ayanova-server[6549]: 2022-09-10 12:26:41.2420|ERROR|Microsoft.EntityFrameworkCore.Database.Connection|An error occurred using the connection to database 'AyaNova' on server ''. -Sep 10 05:26:41 devops ayanova-server[6549]: 2022-09-10 12:26:41.2475|ERROR|Microsoft.EntityFrameworkCore.Query|An exception occurred while iterating over the results of a query for context type 'AyaNova.Models.AyContext'. -Sep 10 05:26:41 devops ayanova-server[6549]: System.InvalidOperationException: An exception has been raised that is likely due to a transient failure. -Sep 10 05:26:41 devops ayanova-server[6549]: ---> Npgsql.NpgsqlException (0x80004005): Failed to connect to 127.0.0.1:5432 -Sep 10 05:26:41 devops ayanova-server[6549]: ---> System.Net.Sockets.SocketException (111): Connection refused - - -Sep 10 05:28:11 devops ayanova-server[6549]: 2022-09-10 12:28:11.3090|ERROR|NotifyEventProcessor|Error adding general notify event [type:ServerOperationsProblem, userId:0, message:Server::ProcessJobsAsync unexpected error during processing -Sep 10 05:28:11 devops ayanova-server[6549]: Exception error: Npgsql.EntityFrameworkCore.PostgreSQL -> An exception has been raised that is likely due to a transient failure. -Sep 10 05:28:11 devops ayanova-server[6549]: Npgsql -> Failed to connect to 127.0.0.1:5432 -Sep 10 05:28:11 devops ayanova-server[6549]: Npgsql -> Connection refused -Sep 10 05:28:11 devops ayanova-server[6549]: ]|System.InvalidOperationException: An exception has been raised that is likely due to a transient failure. - - - -AYANOVA LOG: -snippets to illustrate where this is coming from: - -TOP OF AYANOVA LOG: - -This shows when and where it started: - -root@devops:/var/ayanova/data/logs# ls -log-ayanova-20220906.txt log-ayanova.txt -root@devops:/var/ayanova/data/logs# less log-ayanova.txt -2022-09-07 00:19:34.8404|INFO|AyaNova.Api.Controllers.AuthController|User "AyaNova SuperUser" logged in from "184.67.144.18" ok -2022-09-07 00:19:55.2035|INFO|AyaNova.Api.Controllers.LicenseController|Erasing Database "AyaNova" -2022-09-07 00:19:55.4642|INFO|AyaNova.Api.Controllers.LicenseController|Importing any missing stock Report templates -2022-09-07 00:19:55.6157|INFO|AyaNova.Api.Controllers.LicenseController|Database erase completed -2022-09-07 00:20:21.6220|INFO|AyaNova.Api.Controllers.LicenseController|Fetching license for DBID rVQqoo3YnaKCQDISEkDfYv30XqEl7Keo8bAnsALntM4= -2022-09-07 00:20:22.5560|INFO|AyaNova.Api.Controllers.LicenseController|License - new key installed [TRIAL, regto: GZTestCo-test.onayanova.com, keyid: 1662510016, dbid: rVQqoo3YnaKCQDISEkDfYv30XqEl7Keo8bAnsALntM4=, type: perpetual, exp: 09/12/2022 00:20 (utc), maint. sub. exps: 09/12/2022 00:20 (utc), feat:ServiceTechs - 250] -2022-09-07 00:20:29.1714|INFO|AyaNova.Api.Controllers.AuthController|User "AyaNova SuperUser" logged in from "184.67.144.18" ok -2022-09-07 00:20:36.5056|INFO|Seeder|Seeding data, level Small, time zone offset -7 -2022-09-07 00:20:36.5069|INFO|Seeder|Erasing Database "AyaNova" -2022-09-07 00:20:36.7076|INFO|Seeder|Importing any missing stock Report templates -2022-09-07 00:20:36.7911|INFO|Seeder|Database erase completed -2022-09-07 00:20:40.0636|INFO|Seeder|Seeding SMALL sample data.... -2022-09-07 00:21:31.1362|INFO|Seeder|Small level sample data seeded in 51 seconds -2022-09-07 00:21:31.1802|INFO|Seeder|Seeding completed successfully -2022-09-07 00:21:31.1836|INFO|Seeder|Seeder: setting server state back to Open -2022-09-07 00:22:43.0309|INFO|AyaNova.Api.Controllers.AuthController|User "Eden Collins - BizAdmin" logged in from "184.67.144.18" ok - -BACKUP?? -2022-09-08 22:36:21.4232|ERROR|JobsBiz|Server::ProcessJobsAsync unexpected error during processing=>Npgsql.PostgresException (0x80004005): 57P01: terminating connection due to administrator command - at Npgsql.Internal.NpgsqlConnector.g__ReadMessageLong|211_0(NpgsqlConnector connector, Boolean async, DataRowLoadingMode dataRowLoadingMode, Boolean readingNotifications, Boolean isReadingPrependedMessage) - at Npgsql.NpgsqlDataReader.NextResult(Boolean async, Boolean isConsuming, CancellationToken cancellationToken) - at Npgsql.NpgsqlCommand.ExecuteReader(CommandBehavior behavior, Boolean async, CancellationToken cancellationToken) - at Npgsql.NpgsqlCommand.ExecuteReader(CommandBehavior behavior, Boolean async, CancellationToken cancellationToken) - at Npgsql.NpgsqlCommand.ExecuteDbDataReaderAsync(CommandBehavior behavior, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Storage.RelationalCommand.ExecuteReaderAsync(RelationalCommandParameterObject parameterObject, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Storage.RelationalCommand.ExecuteReaderAsync(RelationalCommandParameterObject parameterObject, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.InitializeReaderAsync(AsyncEnumerator enumerator, CancellationToken cancellationToken) - at Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.NpgsqlExecutionStrategy.ExecuteAsync[TState,TResult](TState state, Func`4 operation, Func`4 verifySucceeded, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.MoveNextAsync() - at Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions.ToListAsync[TSource](IQueryable`1 source, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions.ToListAsync[TSource](IQueryable`1 source, CancellationToken cancellationToken) - at AyaNova.Biz.JobsBiz.GetReadyJobsAsync(Boolean exclusiveOnly) in C:\data\code\raven\server\AyaNova\biz\JobsBiz.cs:line 58 - at AyaNova.Biz.JobsBiz.GetReadyJobsExclusiveOnlyAsync() in C:\data\code\raven\server\AyaNova\biz\JobsBiz.cs:line 30 - at AyaNova.Biz.JobsBiz.ProcessJobsAsync() in C:\data\code\raven\server\AyaNova\biz\JobsBiz.cs:line 255 - Exception data: - Severity: FATAL - SqlState: 57P01 - MessageText: terminating connection due to administrator command - File: postgres.c - Line: 3192 - Routine: ProcessInterrupts -2022-09-08 22:36:21.5122|ERROR|Microsoft.EntityFrameworkCore.Database.Connection|An error occurred using the connection to database 'AyaNova' on server ''. -2022-09-08 22:36:21.5550|ERROR|NotifyEventProcessor|Error adding general notify event [type:ServerOperationsProblem, userId:0, message:Server::ProcessJobsAsync unexpected error during processing -Exception error: Npgsql -> 57P01: terminating connection due to administrator command -]=>System.InvalidOperationException: An exception has been raised that is likely due to a transient failure. - ---> Npgsql.NpgsqlException (0x80004005): Failed to connect to 127.0.0.1:5432 - ---> System.Net.Sockets.SocketException (111): Connection refused - at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.System.Threading.Tasks.Sources.IValueTaskSource.GetResult(Int16 token) - at System.Net.Sockets.Socket.g__WaitForConnectWithCancellation|277_0(AwaitableSocketAsyncEventArgs saea, ValueTask connectTask, CancellationToken cancellationToken) - at Npgsql.TaskExtensions.ExecuteWithTimeout(Func`2 func, NpgsqlTimeout timeout, CancellationToken cancellationToken) - at Npgsql.Internal.NpgsqlConnector.ConnectAsync(NpgsqlTimeout timeout, CancellationToken cancellationToken) - at Npgsql.Internal.NpgsqlConnector.ConnectAsync(NpgsqlTimeout timeout, CancellationToken cancellationToken) - at Npgsql.Internal.NpgsqlConnector.RawOpen(SslMode sslMode, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken, Boolean isFirstAttempt) - at Npgsql.Internal.NpgsqlConnector.g__OpenCore|191_1(NpgsqlConnector conn, SslMode sslMode, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken, Boolean isFirstAttempt) - at Npgsql.Internal.NpgsqlConnector.Open(NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) - at Npgsql.ConnectorPool.OpenNewConnector(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) - at Npgsql.ConnectorPool.g__RentAsync|28_0(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) - at Npgsql.NpgsqlConnection.g__OpenAsync|45_0(Boolean async, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenInternalAsync(Boolean errorsExpected, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenInternalAsync(Boolean errorsExpected, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenAsync(CancellationToken cancellationToken, Boolean errorsExpected) - at Microsoft.EntityFrameworkCore.Storage.RelationalCommand.ExecuteReaderAsync(RelationalCommandParameterObject parameterObject, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.InitializeReaderAsync(AsyncEnumerator enumerator, CancellationToken cancellationToken) - at Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.NpgsqlExecutionStrategy.ExecuteAsync[TState,TResult](TState state, Func`4 operation, Func`4 verifySucceeded, CancellationToken cancellationToken) - --- End of inner exception stack trace --- - at Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.NpgsqlExecutionStrategy.ExecuteAsync[TState,TResult](TState state, Func`4 operation, Func`4 verifySucceeded, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.MoveNextAsync() - at Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions.ToListAsync[TSource](IQueryable`1 source, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions.ToListAsync[TSource](IQueryable`1 source, CancellationToken cancellationToken) - at AyaNova.Biz.NotifyEventHelper.AddGeneralNotifyEvent(AyaType atype, Int64 objectid, NotifyEventType eventType, String message, String name, Exception except, Int64 userId) in C:\data\code\raven\server\AyaNova\biz\NotifyEventHelper.cs:line 376 -2022-09-08 22:36:22.5917|ERROR|Microsoft.EntityFrameworkCore.Database.Connection|An error occurred using the connection to database 'AyaNova' on server ''. -2022-09-08 22:36:22.6209|ERROR|JobsBiz|Server::ProcessJobsAsync unexpected error during processing=>System.InvalidOperationException: An exception has been raised that is likely due to a transient failure. - ---> Npgsql.NpgsqlException (0x80004005): Failed to connect to 127.0.0.1:5432 - ---> System.Net.Sockets.SocketException (111): Connection refused - at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.System.Threading.Tasks.Sources.IValueTaskSource.GetResult(Int16 token) - at System.Net.Sockets.Socket.g__WaitForConnectWithCancellation|277_0(AwaitableSocketAsyncEventArgs saea, ValueTask connectTask, CancellationToken cancellationToken) - at Npgsql.TaskExtensions.ExecuteWithTimeout(Func`2 func, NpgsqlTimeout timeout, CancellationToken cancellationToken) - at Npgsql.Internal.NpgsqlConnector.ConnectAsync(NpgsqlTimeout timeout, CancellationToken cancellationToken) - at Npgsql.Internal.NpgsqlConnector.ConnectAsync(NpgsqlTimeout timeout, CancellationToken cancellationToken) - at Npgsql.Internal.NpgsqlConnector.RawOpen(SslMode sslMode, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken, Boolean isFirstAttempt) - at Npgsql.Internal.NpgsqlConnector.g__OpenCore|191_1(NpgsqlConnector conn, SslMode sslMode, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken, Boolean isFirstAttempt) - at Npgsql.Internal.NpgsqlConnector.Open(NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) - at Npgsql.ConnectorPool.OpenNewConnector(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) - at Npgsql.ConnectorPool.g__RentAsync|28_0(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) - at Npgsql.NpgsqlConnection.g__OpenAsync|45_0(Boolean async, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenInternalAsync(Boolean errorsExpected, CancellationToken cancellationToken) -log-ayanova.txt - at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenInternalAsync(Boolean errorsExpected, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenAsync(CancellationToken cancellationToken, Boolean errorsExpected) - at Microsoft.EntityFrameworkCore.Storage.RelationalCommand.ExecuteReaderAsync(RelationalCommandParameterObject parameterObject, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.InitializeReaderAsync(AsyncEnumerator enumerator, CancellationToken cancellationToken) - at Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.NpgsqlExecutionStrategy.ExecuteAsync[TState,TResult](TState state, Func`4 operation, Func`4 verifySucceeded, CancellationToken cancellationToken) - --- End of inner exception stack trace --- - at Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.NpgsqlExecutionStrategy.ExecuteAsync[TState,TResult](TState state, Func`4 operation, Func`4 verifySucceeded, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.MoveNextAsync() - at Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions.ToListAsync[TSource](IQueryable`1 source, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions.ToListAsync[TSource](IQueryable`1 source, CancellationToken cancellationToken) - at AyaNova.Biz.JobsBiz.GetReadyJobsAsync(Boolean exclusiveOnly) in C:\data\code\raven\server\AyaNova\biz\JobsBiz.cs:line 58 - at AyaNova.Biz.JobsBiz.GetReadyJobsExclusiveOnlyAsync() in C:\data\code\raven\server\AyaNova\biz\JobsBiz.cs:line 30 - at AyaNova.Biz.JobsBiz.ProcessJobsAsync() in C:\data\code\raven\server\AyaNova\biz\JobsBiz.cs:line 255 -2022-09-08 22:36:22.6692|ERROR|Microsoft.EntityFrameworkCore.Database.Connection|An error occurred using the connection to database 'AyaNova' on server ''. -2022-09-08 22:36:22.7000|ERROR|NotifyEventProcessor|Error adding general notify event [type:ServerOperationsProblem, userId:0, message:Server::ProcessJobsAsync unexpected error during processing -Exception error: Npgsql.EntityFrameworkCore.PostgreSQL -> An exception has been raised that is likely due to a transient failure. -Npgsql -> Failed to connect to 127.0.0.1:5432 -System.Net.Sockets -> Connection refused -]=>System.InvalidOperationException: An exception has been raised that is likely due to a transient failure. - ---> Npgsql.NpgsqlException (0x80004005): Failed to connect to 127.0.0.1:5432 - ---> System.Net.Sockets.SocketException (111): Connection refused - at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.CreateException(SocketError error, Boolean forAsyncThrow) - at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.ConnectAsync(Socket socket) - at System.Net.Sockets.Socket.ConnectAsync(EndPoint remoteEP, CancellationToken cancellationToken) - at Npgsql.Internal.NpgsqlConnector.<>c__DisplayClass198_0.b__4(CancellationToken ct) - at Npgsql.TaskExtensions.ExecuteWithTimeout(Func`2 func, NpgsqlTimeout timeout, CancellationToken cancellationToken) - at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) - at Npgsql.TaskExtensions.ExecuteWithTimeout(Func`2 func, NpgsqlTimeout timeout, CancellationToken cancellationToken) - at Npgsql.Internal.NpgsqlConnector.ConnectAsync(NpgsqlTimeout timeout, CancellationToken cancellationToken) - at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) - at Npgsql.Internal.NpgsqlConnector.ConnectAsync(NpgsqlTimeout timeout, CancellationToken cancellationToken) - at Npgsql.Internal.NpgsqlConnector.RawOpen(SslMode sslMode, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken, Boolean isFirstAttempt) - at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) - at Npgsql.Internal.NpgsqlConnector.RawOpen(SslMode sslMode, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken, Boolean isFirstAttempt) - at Npgsql.Internal.NpgsqlConnector.g__OpenCore|191_1(NpgsqlConnector conn, SslMode sslMode, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken, Boolean isFirstAttempt) - at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) - at Npgsql.Internal.NpgsqlConnector.Open(NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) - at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) - at Npgsql.Internal.NpgsqlConnector.Open(NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) - at Npgsql.ConnectorPool.OpenNewConnector(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) - at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) - at Npgsql.ConnectorPool.OpenNewConnector(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) - at Npgsql.ConnectorPool.g__RentAsync|28_0(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) - at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) - at Npgsql.ConnectorPool.g__RentAsync|28_0(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) - at Npgsql.ConnectorPool.Get(NpgsqlConnection conn, NpgsqlTimeout timeout, Boolean async, CancellationToken cancellationToken) - at Npgsql.NpgsqlConnection.g__OpenAsync|45_0(Boolean async, CancellationToken cancellationToken) - at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) - at Npgsql.NpgsqlConnection.g__OpenAsync|45_0(Boolean async, CancellationToken cancellationToken) - at Npgsql.NpgsqlConnection.Open(Boolean async, CancellationToken cancellationToken) - at Npgsql.NpgsqlConnection.OpenAsync(CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenDbConnectionAsync(Boolean errorsExpected, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenInternalAsync(Boolean errorsExpected, CancellationToken cancellationToken) - at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) - at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenInternalAsync(Boolean errorsExpected, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenAsync(CancellationToken cancellationToken, Boolean errorsExpected) - at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) - at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenAsync(CancellationToken cancellationToken, Boolean errorsExpected) - at Microsoft.EntityFrameworkCore.Storage.RelationalCommand.ExecuteReaderAsync(RelationalCommandParameterObject parameterObject, CancellationToken cancellationToken) - at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) - at Microsoft.EntityFrameworkCore.Storage.RelationalCommand.ExecuteReaderAsync(RelationalCommandParameterObject parameterObject, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.InitializeReaderAsync(AsyncEnumerator enumerator, CancellationToken cancellationToken) - at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) - at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.InitializeReaderAsync(AsyncEnumerator enumerator, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.<>c.b__18_0(DbContext _, AsyncEnumerator enumerator, CancellationToken cancellationToken) - at Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.NpgsqlExecutionStrategy.ExecuteAsync[TState,TResult](TState state, Func`4 operation, Func`4 verifySucceeded, CancellationToken cancellationToken) - at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) - at Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.NpgsqlExecutionStrategy.ExecuteAsync[TState,TResult](TState state, Func`4 operation, Func`4 verifySucceeded, CancellationToken cancellationToken) - at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.MoveNextAsync() - at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) - at Microsoft.EntityFrameworkCore.Query.Internal.SingleQueryingEnumerable`1.AsyncEnumerator.MoveNextAsync() - at System.Runtime.CompilerServices.ConfiguredCancelableAsyncEnumerable`1.Enumerator.MoveNextAsync() - at Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions.ToListAsync[TSource](IQueryable`1 source, CancellationToken cancellationToken) - at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) - at Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions.ToListAsync[TSource](IQueryable`1 source, CancellationToken cancellationToken) - at AyaNova.Biz.NotifyEventHelper.AddGeneralNotifyEvent(AyaType atype, Int64 objectid, NotifyEventType eventType, String message, String name, Exception except, Int64 userId) in C:\data\code\raven\server\AyaNova\biz\NotifyEventHelper.cs:line 376 - at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) - at AyaNova.Biz.NotifyEventHelper.AddGeneralNotifyEvent(AyaType atype, Int64 objectid, NotifyEventType eventType, String message, String name, Exception except, Int64 userId) - at AyaNova.Biz.NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType eventType, String message, String name, Exception except, Int64 userId) in C:\data\code\raven\server\AyaNova\biz\NotifyEventHelper.cs:line 311 - at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) - at AyaNova.Biz.NotifyEventHelper.AddGeneralNotifyEvent(NotifyEventType eventType, String message, String name, Exception except, Int64 userId) - at AyaNova.Biz.NotifyEventHelper.AddOpsProblemEvent(String message, Exception ex) in C:\data\code\raven\server\AyaNova\biz\NotifyEventHelper.cs:line 300 - at System.Runtime.CompilerServices.AsyncMethodBuilderCore.Start[TStateMachine](TStateMachine& stateMachine) - at AyaNova.Biz.NotifyEventHelper.AddOpsProblemEvent(String message, Exception ex) - at AyaNova.Biz.JobsBiz.ProcessJobsAsync() in C:\data\code\raven\server\AyaNova\biz\JobsBiz.cs:line 309 - - - - -POSTGRES LOG: -root@devops:/var/log/postgresql# - -2022-09-06 22:11:18.105 UTC [4383] LOG: starting PostgreSQL 14.5 (Ubuntu 14.5-1.pgdg22.04+1) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 11.2.0-19ubuntu1) 11.2.0, 64-bit -2022-09-06 22:11:18.106 UTC [4383] LOG: listening on IPv6 address "::1", port 5432 -2022-09-06 22:11:18.106 UTC [4383] LOG: listening on IPv4 address "127.0.0.1", port 5432 -2022-09-06 22:11:18.107 UTC [4383] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432" -2022-09-06 22:11:18.112 UTC [4384] LOG: database system was shut down at 2022-09-06 22:11:16 UTC -2022-09-06 22:11:18.120 UTC [4383] LOG: database system is ready to accept connections -2022-09-06 22:16:32.257 UTC [6567] postgres@AyaNova FATAL: database "AyaNova" does not exist -2022-09-06 22:16:35.534 UTC [6571] postgres@AyaNova ERROR: relation "aschemaversion" does not exist at character 28 -2022-09-06 22:16:35.534 UTC [6571] postgres@AyaNova STATEMENT: SELECT a.id, a.schema - FROM aschemaversion AS a - LIMIT 2 -2022-09-08 22:36:20.560 UTC [4383] LOG: received fast shutdown request -2022-09-08 22:36:20.564 UTC [4383] LOG: aborting any active transactions -2022-09-08 22:36:20.565 UTC [28038] postgres@AyaNova FATAL: terminating connection due to administrator command -2022-09-08 22:36:20.594 UTC [4383] LOG: background worker "logical replication launcher" (PID 4390) exited with exit code 1 -2022-09-08 22:36:20.595 UTC [4385] LOG: shutting down -2022-09-08 22:36:20.647 UTC [4383] LOG: database system is shut down -