Files
raven/server/AyaNova/util/DbUtil.cs
2021-05-31 15:15:37 +00:00

815 lines
32 KiB
C#

using System;
using Microsoft.Extensions.Logging;
using AyaNova.Models;
using System.Collections.Generic;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using System.Linq;
namespace AyaNova.Util
{
internal static class DbUtil
{
private static string _RawAyaNovaConnectionString;
private static string _dbConnectionString;
private static string _dbName;
private static string _dbUserName;
private static string _dbPassword;
private static string _dbServer;
#region parse connection string
internal static void ParseConnectionString(ILogger _log, string AyaNovaConnectionString)
{
if (string.IsNullOrWhiteSpace(AyaNovaConnectionString))
{
_log.LogDebug("There is no database server connection string set, AYANOVA_DB_CONNECTION is missing or empty. Will use default: \"Server=localhost;Username=postgres;Database=AyaNova;\"");
AyaNovaConnectionString = "Server=localhost;Username=postgres;Database=AyaNova;";
}
_RawAyaNovaConnectionString = AyaNovaConnectionString;
var builder = new System.Data.Common.DbConnectionStringBuilder();
builder.ConnectionString = AyaNovaConnectionString;
if (!builder.ContainsKey("database"))
{
_log.LogDebug("There is no database name specified (\"Database=<NAME>\") in connection string. Will use default: \"Database=AyaNova;\"");
builder.Add("database", "AyaNova");
}
//Keep track of default values
_dbConnectionString = builder.ConnectionString;
if (builder.ContainsKey("database"))
_dbName = builder["database"].ToString();
if (builder.ContainsKey("username"))
_dbUserName = builder["username"].ToString();
if (builder.ContainsKey("password"))
_dbPassword = builder["password"].ToString();
if (builder.ContainsKey("server"))
_dbServer = builder["server"].ToString();
_log.LogDebug("AyaNova will use the following connection string: {0}", PasswordRedactedConnectionString(_dbConnectionString));
}
///////////////////////////////////////////
//clean out password from connection string
//for log purposes
internal static string PasswordRedactedConnectionString(string cs)
{
var nStart = 0;
var nStop = 0;
var lwrcs = cs.ToLowerInvariant();
nStart = lwrcs.IndexOf("password");
if (nStart == -1)
{
//no password, just return it
return cs;
}
//find terminating semicolon
nStop = lwrcs.IndexOf(";", nStart);
if (nStop == -1 || nStop == lwrcs.Length)
{
//no terminating semicolon or that is the final character in the string
return cs.Substring(0, nStart + 9) + "[redacted];";
}
else
{
//not the last thing in the string so return the whole string minus the password part
return cs.Substring(0, nStart + 9) + "[redacted];" + cs.Substring(nStop + 1);
}
}
#endregion
#region Connection utilities
///////////////////////////////////////////
//Verify that server exists
//
private static string AdminConnectionString
{
get
{
return _dbConnectionString.Replace(_dbName, "postgres");
}
}
///////////////////////////////////////////
//Connection string without password
//
internal static string DisplayableConnectionString
{
get
{
return PasswordRedactedConnectionString(_dbConnectionString);
}
}
#endregion
#region DB verification
///////////////////////////////////////////
// Get database server version
//
internal static string DBServerVersion(AyaNova.Models.AyContext ct)
{
using (var cmd = ct.Database.GetDbConnection().CreateCommand())
{
ct.Database.OpenConnection();
cmd.CommandText = $"select version();";
using (var dr = cmd.ExecuteReader())
{
if (dr.Read())
{
if (dr.IsDBNull(0))
return "Unknown / no results";
else
return (dr.GetString(0));
}
else
{
return "Unknown / no results";
}
}
}
}
///////////////////////////////////////////
//Verify that server exists
// spend up to 5 minutes waiting for it to come up before bailing
//
internal static bool DatabaseServerExists(ILogger log, string logPrepend)
{
try
{
//Try every 5 seconds for 60 tries before giving up (5 minutes total)
var maxRetryAttempts = 60;
var pauseBetweenFailures = TimeSpan.FromSeconds(5);
RetryHelper.RetryOnException(maxRetryAttempts, pauseBetweenFailures, log, logPrepend + DisplayableConnectionString, () =>
{
using (var conn = new Npgsql.NpgsqlConnection(AdminConnectionString))
{
conn.Open();
conn.Close();
}
});
}
catch
{
return false;
}
return true;
}
///////////////////////////////////////////
//Verify that database exists, if not, then create it
//
internal static bool EnsureDatabaseExists(ILogger _log)
{
_log.LogDebug("Ensuring database exists. Connection string is: \"{0}\"", DisplayableConnectionString);
using (var conn = new Npgsql.NpgsqlConnection(_dbConnectionString))
{
try
{
conn.Open();
conn.Close();
}
catch (Exception e)
{
//if it's a db doesn't exist that's ok, we'll create it, not an error
if (e is Npgsql.PostgresException)
{
if (((Npgsql.PostgresException)e).SqlState == "3D000")
{
//create the db here
using (var cnCreate = new Npgsql.NpgsqlConnection(AdminConnectionString))
{
cnCreate.Open();
// Create the database desired
using (var cmd = new Npgsql.NpgsqlCommand())
{
cmd.Connection = cnCreate;
cmd.CommandText = "CREATE DATABASE \"" + _dbName + "\";";
cmd.ExecuteNonQuery();
_log.LogInformation("Database \"{0}\" created successfully!", _dbName);
}
cnCreate.Close();
}
}
else
{
var err = string.Format("Database server connection failed. Connection string is: \"{0}\"", DisplayableConnectionString);
_log.LogCritical(e, "BOOT: E1000 - " + err);
err = err + "\nError reported was: " + e.Message;
throw new ApplicationException(err);
}
}
}
}
return true;
}
#endregion
#region DB utilities
///////////////////////////////////////////
// Drop and re-create db
// This is the NUCLEAR option and
// completely ditches the DB and all user uploaded files
//
internal static async Task DropAndRecreateDbAsync(ILogger _log)
{
_log.LogInformation("Dropping and creating Database \"{0}\"", _dbName);
//clear all connections so that the database can be dropped
Npgsql.NpgsqlConnection.ClearAllPools();
using (var conn = new Npgsql.NpgsqlConnection(AdminConnectionString))
{
await conn.OpenAsync();
// Create the database desired
using (var cmd = new Npgsql.NpgsqlCommand())
{
cmd.Connection = conn;
cmd.CommandText = "DROP DATABASE \"" + _dbName + "\";";
await cmd.ExecuteNonQueryAsync();
cmd.Connection = conn;
cmd.CommandText = "CREATE DATABASE \"" + _dbName + "\";";
await cmd.ExecuteNonQueryAsync();
_log.LogDebug("Database created");
}
await conn.CloseAsync();
}
//final cleanup step is to erase user uploaded files
FileUtil.EraseEntireContentsOfAttachmentFilesFolder();
}
/////////////////////////////////////////////////////////
// Erase all user entered data from the db
// This is called by seeder for trial seeding purposes
// and by v8 migrate v7 exporter
internal static async Task EmptyBizDataFromDatabaseForSeedingOrImportingAsync(ILogger _log)
{
_log.LogInformation("Erasing Database \"{0}\"", _dbName);
AyaNova.Api.ControllerHelpers.ApiServerState apiServerState = (AyaNova.Api.ControllerHelpers.ApiServerState)ServiceProviderProvider.Provider.GetService(typeof(AyaNova.Api.ControllerHelpers.ApiServerState));
apiServerState.SetClosed("Erasing database");
//clear all connections so that the database can be dropped
Npgsql.NpgsqlConnection.ClearAllPools();
using (var conn = new Npgsql.NpgsqlConnection(_dbConnectionString))
{
await conn.OpenAsync();
//### DELIBERATELY IGNORED
//Some data is deliberately not deleted for now:
//Reports
//Logos
//prepare to delete by removing foreign keys
using (var cmd = new Npgsql.NpgsqlCommand())
{
cmd.Connection = conn;
cmd.CommandText = "update auser set customerid=null;";
await cmd.ExecuteNonQueryAsync();
cmd.Connection = conn;
cmd.CommandText = "update aunit set replacedbyunitid=null, parentunitid=null, purchasedfromvendorid=null, unitmodelid=null;";
await cmd.ExecuteNonQueryAsync();
cmd.CommandText = "update auser set headofficeid=null;";
await cmd.ExecuteNonQueryAsync();
cmd.CommandText = "update auser set vendorid=null;";
await cmd.ExecuteNonQueryAsync();
cmd.CommandText = "update aloanunit set unitid=null, workorderitemloanid=null;";
await cmd.ExecuteNonQueryAsync();
cmd.CommandText = "update aglobalbizsettings set taxpartpurchaseid=null,taxpartsaleid=null,taxratesaleid=null;";
await cmd.ExecuteNonQueryAsync();
}
//Delete non stock translations
using (var cmd = new Npgsql.NpgsqlCommand())
{
cmd.Connection = conn;
//set to default translation so can delete all non default ones
cmd.CommandText = "update auseroptions set translationid=1;";
await cmd.ExecuteNonQueryAsync();
cmd.CommandText = "delete from atranslationitem where translationid > 4;";
await cmd.ExecuteNonQueryAsync();
cmd.CommandText = "delete from atranslation where id > 4;";
await cmd.ExecuteNonQueryAsync();
}
//REMOVE ALL REMAINING DATA
//--- WorkOrder
await EraseTableAsync("aworkorderitemexpense", conn);
await EraseTableAsync("aworkorderitemlabor", conn);
await EraseTableAsync("aworkorderitemloan", conn);
await EraseTableAsync("aworkorderitempart", conn);
await EraseTableAsync("aworkorderitempartrequest", conn);
await EraseTableAsync("aworkorderitemscheduleduser", conn);
await EraseTableAsync("aworkorderitemtask", conn);
await EraseTableAsync("aworkorderitemtravel", conn);
await EraseTableAsync("aworkorderitemunit", conn);
await EraseTableAsync("aworkorderitemoutsideservice", conn);
await EraseTableAsync("aworkorderitem", conn);
await EraseTableAsync("aworkorderstate", conn);
await EraseTableAsync("aworkorder", conn);
await EraseTableAsync("aworkordertemplateitem", conn);
await EraseTableAsync("aworkordertemplate", conn);
//---
await EraseTableAsync("afileattachment", conn);
await EraseTableAsync("acustomerservicerequest", conn);
await EraseTableAsync("awidget", conn);
await EraseTableAsync("aevent", conn);
await EraseTableAsync("adatalistsavedfilter", conn);
await EraseTableAsync("adatalistcolumnview", conn);
await EraseTableAsync("apicklisttemplate", conn, true);
await EraseTableAsync("aformcustom", conn);
await EraseTableAsync("asearchkey", conn);
await EraseTableAsync("asearchdictionary", conn);
await EraseTableAsync("atag", conn);
await EraseTableAsync("apurchaseorder", conn);
await EraseTableAsync("aloanunit", conn);
await EraseTableAsync("apartassemblyitem", conn);
await EraseTableAsync("apartassembly", conn);
await EraseTableAsync("apartinventory", conn);
await EraseTableAsync("apart", conn);
await EraseTableAsync("apmitem", conn);
await EraseTableAsync("apm", conn);
await EraseTableAsync("apmtemplateitem", conn);
await EraseTableAsync("apmtemplate", conn);
await EraseTableAsync("aquoteitem", conn);
await EraseTableAsync("aquote", conn);
await EraseTableAsync("aquotetemplateitem", conn);
await EraseTableAsync("aquotetemplate", conn);
await EraseTableAsync("aunitmodel", conn);
await EraseTableAsync("avendor", conn);
await EraseTableAsync("aunit", conn);
await EraseTableAsync("aproject", conn);//depends on User, dependants are wo,quote,pm
await EraseTableAsync("acustomernote", conn);
await EraseTableAsync("acustomer", conn);
await EraseTableAsync("aheadoffice", conn);
await EraseTableAsync("acontract", conn);
//----- NOTIFICATION
await EraseTableAsync("anotification", conn);
await EraseTableAsync("anotifyevent", conn);
await EraseTableAsync("anotifydeliverylog", conn);
await EraseTableAsync("anotifysubscription", conn);
await EraseTableAsync("amemo", conn);
await EraseTableAsync("areminder", conn);//depends on User
await EraseTableAsync("areview", conn);//depends on User
await EraseTableAsync("aservicerate", conn);
await EraseTableAsync("atravelrate", conn);
await EraseTableAsync("ataxcode", conn);
await EraseTableAsync("aservicebank", conn);
await EraseTableAsync("aworkorderstatus", conn);
await EraseTableAsync("aworkorderitemstatus", conn);
await EraseTableAsync("aworkorderitempriority", conn);
await EraseTableAsync("ataskgroup", conn);//items cascade
//after cleanup
using (var cmd = new Npgsql.NpgsqlCommand())
{
cmd.Connection = conn;
cmd.CommandText = "delete from \"auseroptions\" where UserId <> 1;";
await cmd.ExecuteNonQueryAsync();
cmd.CommandText = "ALTER SEQUENCE auseroptions_id_seq RESTART WITH 2;";
await cmd.ExecuteNonQueryAsync();
cmd.CommandText = "delete from \"auser\" where id <> 1;";
await cmd.ExecuteNonQueryAsync();
cmd.CommandText = "ALTER SEQUENCE auser_id_seq RESTART WITH 2;";
await cmd.ExecuteNonQueryAsync();
cmd.CommandText = "delete from \"adashboardview\" where userid <> 1;";
await cmd.ExecuteNonQueryAsync();
cmd.CommandText = $"ALTER SEQUENCE adashboardview_id_seq RESTART WITH 2;";
await cmd.ExecuteNonQueryAsync();
cmd.CommandText = "delete from \"apartwarehouse\" where id <> 1;";
await cmd.ExecuteNonQueryAsync();
cmd.CommandText = $"ALTER SEQUENCE apartwarehouse_id_seq RESTART WITH 2;";
await cmd.ExecuteNonQueryAsync();
cmd.CommandText = "ALTER SEQUENCE apurchaseorder_serial_seq RESTART WITH 1;";
await cmd.ExecuteNonQueryAsync();
cmd.CommandText = "ALTER SEQUENCE aworkorder_serial_seq RESTART WITH 1;";
await cmd.ExecuteNonQueryAsync();
cmd.CommandText = "ALTER SEQUENCE aquote_serial_seq RESTART WITH 1;";
await cmd.ExecuteNonQueryAsync();
cmd.CommandText = "ALTER SEQUENCE apm_serial_seq RESTART WITH 1;";
await cmd.ExecuteNonQueryAsync();
}
await conn.CloseAsync();
}
//If we got here then it's safe to erase the attachment files
FileUtil.EraseEntireContentsOfAttachmentFilesFolder();
apiServerState.ResumePriorState();
_log.LogInformation("Database erase completed");
}
///////////////////////////////////////////
// Erase all data from the table specified
//
private static async Task EraseTableAsync(string sTable, Npgsql.NpgsqlConnection conn, bool tableHasNoIdentity = false)
{
using (var cmd = new Npgsql.NpgsqlCommand())
{
cmd.Connection = conn;
//Boo! Can't do this becuase it will fail if there is a foreign key which nearly all tables have unless cascade option is used
//but then cascade causes things to delete in any referenced table
// cmd.CommandText = "TRUNCATE \"" + sTable + "\" RESTART IDENTITY;";
// cmd.CommandText = $"delete from {sTable};";
if (tableHasNoIdentity)
cmd.CommandText = $"TRUNCATE {sTable};";
else
cmd.CommandText = $"TRUNCATE {sTable} RESTART IDENTITY;";
await cmd.ExecuteNonQueryAsync();
// if (!tableHasNoSequence)
// {
// cmd.CommandText = $"ALTER SEQUENCE {sTable}_id_seq RESTART WITH 1;";
// await cmd.ExecuteNonQueryAsync();
// }
}
}
///////////////////////////////////////////
// Check if DB is empty
// CALLED BY LICENSE CONTROLLER AND LICENSE.CS FOR TRIAL Request check
internal static async Task<bool> DBIsEmptyAsync(AyContext ct, ILogger _log)
{
//For efficiency just check a few main tables just stuff that would be shitty to have to re-enter
//Mostly user, customer and vendor cover it because nearly everything else requires those to have any sort of data at all
_log.LogDebug("DB empty check");
//An empty db contains only one User
if (await ct.User.LongCountAsync() > 1) return false;
if (await ct.Customer.AnyAsync()) return false;
if (await ct.Vendor.AnyAsync()) return false;
if (await ct.WorkOrder.AnyAsync()) return false;
if (await ct.Quote.AnyAsync()) return false;
if (await ct.PM.AnyAsync()) return false;
if (await ct.Unit.AnyAsync()) return false;
if (await ct.HeadOffice.AnyAsync()) return false;
if (await ct.LoanUnit.AnyAsync()) return false;
if (await ct.Part.AnyAsync()) return false;
if (await ct.Project.AnyAsync()) return false;
if (await ct.PurchaseOrder.AnyAsync()) return false;
return true;
}
///////////////////////////////////////////
// Check if DB has evaluation user accounts
// CALLED BY by login ping from licent via notify controller
internal static async Task<bool> DBHasTrialUsersAsync(AyContext ct, ILogger _log)
{
_log.LogDebug("DB trial users presence check");
//There are 22 trial users (more but for internal use) in a trial database
if (await ct.User.LongCountAsync() < 22) return false;
//just check for a few for testing
if (await ct.User.AsNoTracking()
.Where(z =>
z.Login == "BizAdminFull" ||
z.Login == "DispatchFull" ||
z.Login == "InventoryFull" ||
z.Login == "Accounting" ||
z.Login == "TechFull"
).LongCountAsync() < 5) return false;
return true;
}
///////////////////////////////////////////
// Ensure the db is not modified
//
internal static async Task CheckFingerPrintAsync(
long ExpectedColumns,
long ExpectedIndexes,
long ExpectedCheckConstraints,
long ExpectedForeignKeyConstraints,
long ExpectedViews,
long ExpectedRoutines,
ILogger _log)
{
_log.LogDebug("Checking DB integrity");
long actualColumns = 0;
long actualIndexes = 0;
long actualCheckConstraints = 0;
long actualForeignKeyConstraints = 0;
long actualViews = 0;
long actualRoutines = 0;
//COLUMNS
using (var conn = new Npgsql.NpgsqlConnection(_dbConnectionString))
{
await conn.OpenAsync();
using (var command = conn.CreateCommand())
{
//Count all columns in all our tables
command.CommandText = "SELECT count(*) FROM information_schema.columns where table_schema='public'";
using (var result = await command.ExecuteReaderAsync())
{
if (result.HasRows)
{
//check the values
await result.ReadAsync();
actualColumns = result.GetInt64(0);
}
else
{
var err = "E1030 - Database integrity check failed, could not obtain COLUMN data. Contact support.";
_log.LogCritical(err);
throw new ApplicationException(err);
}
}
}
//INDEXES
using (var command = conn.CreateCommand())
{
//Count all indexes in all our tables
command.CommandText = "select Count(*) from pg_indexes where schemaname='public'";
using (var result = await command.ExecuteReaderAsync())
{
if (result.HasRows)
{
//check the values
await result.ReadAsync();
actualIndexes = result.GetInt64(0);
}
else
{
var err = "E1030 - Database integrity check failed, could not obtain INDEX data. Contact support.";
_log.LogCritical(err);
throw new ApplicationException(err);
}
}
}
//CHECK CONSTRAINTS
using (var command = conn.CreateCommand())
{
command.CommandText = "SELECT count(*) FROM information_schema.check_constraints where constraint_schema='public'";
using (var result = await command.ExecuteReaderAsync())
{
if (result.HasRows)
{
//check the values
await result.ReadAsync();
actualCheckConstraints = result.GetInt64(0);
}
else
{
var err = "E1030 - Database integrity check failed, could not obtain CHECK CONSTRAINT data. Contact support.";
_log.LogCritical(err);
throw new ApplicationException(err);
}
}
}
//FOREIGN KEY CONSTRAINTS
using (var command = conn.CreateCommand())
{
command.CommandText = "SELECT count(*) FROM information_schema.referential_constraints where constraint_schema='public'";
using (var result = await command.ExecuteReaderAsync())
{
if (result.HasRows)
{
//check the values
await result.ReadAsync();
actualForeignKeyConstraints = result.GetInt64(0);
}
else
{
var err = "E1030 - Database integrity check failed, could not obtain FOREIGN KEY CONSTRAINT data. Contact support.";
_log.LogCritical(err);
throw new ApplicationException(err);
}
}
}
//VIEWS
using (var command = conn.CreateCommand())
{
command.CommandText = "SELECT count(*) FROM information_schema.views where table_schema='public'";
using (var result = await command.ExecuteReaderAsync())
{
if (result.HasRows)
{
//check the values
await result.ReadAsync();
actualViews = result.GetInt64(0);
}
else
{
var err = "E1030 - Database integrity check failed, could not obtain VIEW data. Contact support.";
_log.LogCritical(err);
throw new ApplicationException(err);
}
}
}
//ROUTINES
using (var command = conn.CreateCommand())
{
command.CommandText = "SELECT count(*) FROM information_schema.routines where routine_schema='public'";
using (var result = await command.ExecuteReaderAsync())
{
if (result.HasRows)
{
//check the values
await result.ReadAsync();
actualRoutines = result.GetInt64(0);
}
else
{
var err = "E1030 - Database integrity check failed, could not obtain ROUTINE data. Contact support.";
_log.LogCritical(err);
throw new ApplicationException(err);
}
}
}
await conn.CloseAsync();
if (ExpectedColumns != actualColumns
|| ExpectedIndexes != actualIndexes
|| ExpectedCheckConstraints != actualCheckConstraints
|| ExpectedForeignKeyConstraints != actualForeignKeyConstraints
|| ExpectedRoutines != actualRoutines
|| ExpectedViews != actualViews)
{
var err = $"E1030 - Database integrity check failed (C{actualColumns}:I{actualIndexes}:CC{actualCheckConstraints}:FC{actualForeignKeyConstraints}:V{actualViews}:R{actualRoutines})";
_log.LogCritical(err);
throw new ApplicationException(err);
}
return;
}
}
///////////////////////////////////////////
// Given a table name return the count of records in that table
// Used for metrics
//
///
internal static async Task<long> CountOfRecordsAsync(string TableName)
{
long ret = 0;
using (var conn = new Npgsql.NpgsqlConnection(_dbConnectionString))
{
await conn.OpenAsync();
using (var command = conn.CreateCommand())
{
command.CommandText = $"SELECT count(*) FROM {TableName}";
using (var result = await command.ExecuteReaderAsync())
{
if (result.HasRows)
{
await result.ReadAsync();
ret = result.GetInt64(0);
}
}
}
await conn.CloseAsync();
}
return ret;
}
///////////////////////////////////////////
// Returns all table names that are ours in current schema
//
///
internal static async Task<List<string>> GetAllTablenamesAsync()
{
List<string> ret = new List<string>();
using (var conn = new Npgsql.NpgsqlConnection(_dbConnectionString))
{
await conn.OpenAsync();
using (var command = conn.CreateCommand())
{
command.CommandText = "SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';";
using (var result = await command.ExecuteReaderAsync())
{
if (result.HasRows)
{
while (await result.ReadAsync())
{
ret.Add(result.GetString(0));
}
}
}
}
await conn.CloseAsync();
}
return ret;
}
#endregion
}//eoc
}//eons