raven/server/AyaNova/Startup.cs

using System.IO;
using System.Reflection;
using System.Linq;
using System;
using System.Collections.Generic;
using Microsoft.AspNetCore.Builder;
using Asp.Versioning.ApiExplorer;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.EntityFrameworkCore;
using Microsoft.AspNetCore.Authentication.JwtBearer;
using Microsoft.IdentityModel.Tokens;
using Microsoft.Extensions.Hosting;
using Microsoft.AspNetCore.Http;
using Microsoft.OpenApi;
using Microsoft.Extensions.Options;
using Swashbuckle.AspNetCore.SwaggerGen;

using AyaNova.Models;
using AyaNova.Util;
using AyaNova.Generator;
using AyaNova.Biz;
using NLog.Extensions.Logging;


namespace AyaNova
{
    public class Startup
    {
        /////////////////////////////////////////////////////////////
        //
        public Startup(Microsoft.AspNetCore.Hosting.IWebHostEnvironment hostingEnvironment)
        {
            var nlogLoggerProvider = new NLogLoggerProvider();
            _newLog = nlogLoggerProvider.CreateLogger("SERVER");
            _hostingEnvironment = hostingEnvironment;
            AyaNova.Util.ApplicationLogging.LoggerProvider = nlogLoggerProvider;
            ServerBootConfig.AYANOVA_CONTENT_ROOT_PATH = hostingEnvironment.ContentRootPath;
            ServerBootConfig.SEEDING = false;
        }

        private readonly ILogger _newLog;
        private static int _untrustedProxyWarned = 0;
        private string _connectionString = "";
        private readonly Microsoft.AspNetCore.Hosting.IWebHostEnvironment _hostingEnvironment;

        //checkpoint commit comment for case 4642 to test GIT and build environment etc

        ////////////////////////////////////////////////////////////
        // 
        //
        public void ConfigureServices(IServiceCollection services)
        {
            _newLog.LogDebug("Initializing services...");

            _newLog.LogDebug("Health");
            services.AddHealthChecks().AddDbContextCheck<AyContext>(); ;

            //Server state service for shutting people out of api
            _newLog.LogDebug("ServerState service");
            services.AddSingleton(new AyaNova.Api.ControllerHelpers.ApiServerState());


            _newLog.LogDebug("Mail service");
            services.AddSingleton<IMailer, Mailer>();

            //Init controllers
            _newLog.LogDebug("Controllers");
            var MvcBuilder = services.AddControllers(config =>
                          {
                              // config.Filters.Add(new AyaNova.Api.ControllerHelpers.ApiCustomExceptionFilter(AyaNova.Util.ApplicationLogging.LoggerFactory));
                              config.Filters.Add(new AyaNova.Api.ControllerHelpers.ApiCustomExceptionFilter(_newLog));
                          });




            //Prevent default model binding automatic 400 page so we can consistently show *our* error to our specs
            //https://docs.microsoft.com/en-us/aspnet/core/web-api/index?view=aspnetcore-3.1#automatic-http-400-responses
            MvcBuilder.ConfigureApiBehaviorOptions(options =>
                    {
                        options.SuppressModelStateInvalidFilter = true;
                    });

            _newLog.LogDebug("JSON");

            MvcBuilder.AddNewtonsoftJson(options =>
                {
                    options.SerializerSettings.Formatting = Newtonsoft.Json.Formatting.None;
                    options.SerializerSettings.DateTimeZoneHandling = Newtonsoft.Json.DateTimeZoneHandling.Utc;

                });


            //HTTP CLIENT FACTORY USED BY LICENSE.CS
            //https://docs.microsoft.com/en-us/aspnet/core/fundamentals/http-requests?view=aspnetcore-3.1
            _newLog.LogDebug("HTTPClientFactory");
            services.AddHttpClient();





            //TODO: ENsure report files here

            #region DATABASE


            _connectionString = ServerBootConfig.AYANOVA_DB_CONNECTION;

            //Check DB server exists and can be connected to
            _newLog.LogDebug("Testing database server connection...");

            //parse the connection string properly
            DbUtil.ParseConnectionString(_newLog, _connectionString);

            //Probe for database server
            //Will retry every 5 seconds for up to 5 minutes before bailing
            if (!DbUtil.DatabaseServerExists(_newLog, "Waiting for db server "))
            {
                var err = $"E1000 - AyaNova can't connect to the database server after trying for 5 minutes (connection string is:\"{DbUtil.DisplayableConnectionString}\")";
                _newLog.LogCritical(err);
                throw new System.ApplicationException(err);
            }


            //We have db available
            ServerGlobalOpsSettingsCache.DBAVAILABLE = true;

            _newLog.LogInformation("Connected to database server - {0}", DbUtil.DisplayableConnectionString);



            //ensure database is ready and present
            DbUtil.EnsureDatabaseExists(_newLog);

            bool LOG_SENSITIVE_DATA = false;

#if (DEBUG)
            LOG_SENSITIVE_DATA = false;//############################################################################

#endif

            _newLog.LogDebug("EF Core");


            //change to resolve error:
            //2020-12-28 09:20:14.3545|WARN|Microsoft.EntityFrameworkCore.Infrastructure|'AddEntityFramework*' was called on the service provider, but 'UseInternalServiceProvider' wasn't called in the DbContext options configuration. Consider removing the 'AddEntityFramework*' call, as in most cases it's not needed and may cause conflicts with other products and services registered in the same service provider.
            //https://stackoverflow.com/questions/62917136/addentityframework-was-called-on-the-service-provider-but-useinternalservic
            // services.AddEntityFrameworkNpgsql().AddDbContext<AyContext>(
            //     options => options.UseNpgsql(_connectionString
            //     //,opt => opt.EnableRetryOnFailure()//REMOVED THIS BECAUSE IT WAS INTEFERING WITH TRANSACTIONS BUT THEN DIDN'T USE THE TRANSACTION BUT IT SEEMS FASTER WITHOUT IT AS WELL SO...??
            //     )//http://www.npgsql.org/efcore/misc.html?q=execution%20strategy#execution-strategy
            //                 .ConfigureWarnings(warnings => //https://livebook.manning.com/#!/book/entity-framework-core-in-action/chapter-12/v-10/85
            //                         warnings.Throw( //Throw an exception on client eval, not necessarily an error but a smell
            //                                         // Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.QueryClientEvaluationWarning
            //                         ))
            //                 .EnableSensitiveDataLogging(LOG_SENSITIVE_DATA)
            //     );

            services.AddDbContext<AyContext>(options => options.UseNpgsql(_connectionString).ConfigureWarnings(warnings => warnings.Throw()).EnableSensitiveDataLogging(LOG_SENSITIVE_DATA));

            #endregion


            // Add service and create Policy with options
            _newLog.LogDebug("CORS");
            services.AddCors(options =>
            {
                options.AddPolicy("CorsPolicy",
                    builder => builder.AllowAnyOrigin()
                    .AllowAnyMethod()
                    .AllowAnyHeader().SetPreflightMaxAge(TimeSpan.FromSeconds(600))
                    );
            });




            #region Swagger

            services.AddApiVersioning(options =>
                 {
                     options.AssumeDefaultVersionWhenUnspecified = true;
                     options.DefaultApiVersion = new Asp.Versioning.ApiVersion(8, 0);// Asp.Versioning.ApiVersion..Parse("8.0");
                     options.ReportApiVersions = true;
                 }).AddMvc()//Asp.Versioning 10: register MVC controller versioning explicitly (as in Sockeye), not only through AddApiExplorer
                 .AddApiExplorer(options =>
            {

                // add the versioned api explorer, which also adds IApiVersionDescriptionProvider service
                // note: the specified format code will format the version as "'v'major[.minor][-status]"
                options.GroupNameFormat = "'v'VVV";

                // note: this option is only necessary when versioning by url segment. the SubstitutionFormat
                // can also be used to control the format of the API version in route templates

                //THIS IS WHAT ADDS THE API version PARAMETER AUTOMATICALLY so you don't need to type an 8 in every swagger-ui route test
                options.SubstituteApiVersionInUrl = true;
            });
            services.AddTransient<IConfigureOptions<SwaggerGenOptions>, ConfigureSwaggerOptions>();
            services.AddSwaggerGen(
                c =>
                {
                    // integrate xml comments
                    c.IncludeXmlComments(XmlCommentsFilePath);

                    //https://stackoverflow.com/questions/56234504/migrating-to-swashbuckle-aspnetcore-version-5
                    //First we define the security scheme
                    c.AddSecurityDefinition("Bearer", //Name the security scheme
                        new OpenApiSecurityScheme
                        {
                            Description = "JWT Authorization header using the Bearer scheme.",
                            Type = SecuritySchemeType.Http, //We set the scheme type to http since we're using bearer authentication
                            Scheme = "bearer" //The name of the HTTP Authorization scheme to be used in the Authorization header. In this case "bearer".
                        });

                    //Then require it on every operation, referring to it by the name defined above (OpenAPI.NET 2 / Swashbuckle 10 form)
                    c.AddSecurityRequirement(document => new OpenApiSecurityRequirement
                    {
                        [new OpenApiSecuritySchemeReference("Bearer", document)] = []
                    });
                }
            );


            services.AddSwaggerGenNewtonsoftSupport();

            #endregion

            #region JWT AUTHENTICATION
            //case 4711: the signing key is no longer a setting (placeholders and weak values could be used to forge tokens);
            //it is generated once and kept in a file so sign-ins survive a restart. Deleting the file signs everyone out.
            if (ServerBootConfig.AYANOVA_JWT_SECRET_SETTING_PRESENT)
                _newLog.LogWarning("AYANOVA_JWT_SECRET is no longer used and can be removed from the configuration: AyaNova generates and keeps its own sign-in key");

            ServerBootConfig.AYANOVA_JWT_SECRET = LoadOrCreateJwtSigningKey(ServerBootConfig.JWT_SIGNING_KEY_FILE_PATH);

            string LoadOrCreateJwtSigningKey(string path)
            {
                try
                {
                    if (System.IO.File.Exists(path))
                    {
                        var existing = System.IO.File.ReadAllText(path).Trim();
                        if (existing.Length >= 32)
                            return existing;
                        _newLog.LogWarning($"Sign-in key file {path} is not valid; generating a new one (users will need to log in again)");
                    }
                    var newKey = Convert.ToBase64String(System.Security.Cryptography.RandomNumberGenerator.GetBytes(48));
                    System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(path));
                    var options = new System.IO.FileStreamOptions { Mode = System.IO.FileMode.Create, Access = System.IO.FileAccess.Write };
                    if (!OperatingSystem.IsWindows())
                        options.UnixCreateMode = System.IO.UnixFileMode.UserRead | System.IO.UnixFileMode.UserWrite;
                    using (var writer = new System.IO.StreamWriter(path, System.Text.Encoding.ASCII, options))
                        writer.Write(newKey);
                    _newLog.LogInformation($"Generated a new sign-in key in {path}");
                    return newKey;
                }
                catch (Exception ex)
                {
                    _newLog.LogWarning(ex, $"Could not read or create the sign-in key file {path}; using a temporary key for this run (users will need to log in again after a restart)");
                    return Convert.ToBase64String(System.Security.Cryptography.RandomNumberGenerator.GetBytes(48));
                }
            }

            var signingKey = new SymmetricSecurityKey(System.Text.Encoding.ASCII.GetBytes(ServerBootConfig.AYANOVA_JWT_SECRET));

            _newLog.LogDebug("Authorization");
            services.AddAuthentication(options =>
            {
                options.DefaultAuthenticateScheme = JwtBearerDefaults.AuthenticationScheme;
                options.DefaultChallengeScheme = JwtBearerDefaults.AuthenticationScheme;
            }).AddJwtBearer(options =>
                {
                    // options.AutomaticAuthenticate = true;
                    // options.AutomaticChallenge = true;
                    options.TokenValidationParameters = new TokenValidationParameters
                    {
                        // Token signature will be verified using a private key.
                        ValidateIssuerSigningKey = true,
                        RequireSignedTokens = true,
                        IssuerSigningKey = signingKey,
                        ValidateIssuer = true,
                        ValidIssuer = "ayanova.com",
                        ValidateAudience = false,
                        //ValidAudience = "http://localhost:7575/"

                        // Token will only be valid if not expired yet, with 5 minutes clock skew.
                        ValidateLifetime = true,
                        RequireExpirationTime = true,
                        ClockSkew = TimeSpan.Zero//new TimeSpan(0, 0, 2),
                    };
                });



            #endregion




            _newLog.LogDebug("Generator");
            services.AddSingleton<IHostedService, GeneratorService>();

        }



        ////////////////////////////////////////////////////////////
        // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
        //
        public void Configure(IApplicationBuilder app, Microsoft.AspNetCore.Hosting.IWebHostEnvironment env,
        AyContext dbContext, IApiVersionDescriptionProvider provider, AyaNova.Api.ControllerHelpers.ApiServerState apiServerState,
        IServiceProvider serviceProvider)
        {
            _newLog.LogDebug("Configuring request pipeline...");

            //this *may* be useful in the event of an issue so uncomment if necessary but errors during dev are handled equally by the logging, I think
            // if (env.IsDevelopment())
            // {
            //     app.UseDeveloperExceptionPage();
            // }

            //Store a reference to the dependency injection service for static classes 
            ServiceProviderProvider.Provider = app.ApplicationServices;

            //Enable ability to handle reverse proxy
            //case 4732: a proxy on this computer (loopback) is always trusted; proxies on other computers only when named in AYANOVA_TRUSTED_PROXIES
            var forwardedHeadersOptions = new ForwardedHeadersOptions
            {
                ForwardedHeaders = Microsoft.AspNetCore.HttpOverrides.ForwardedHeaders.XForwardedFor | Microsoft.AspNetCore.HttpOverrides.ForwardedHeaders.XForwardedProto
            };
            if (!string.IsNullOrWhiteSpace(ServerBootConfig.AYANOVA_TRUSTED_PROXIES))
            {
                foreach (var entry in ServerBootConfig.AYANOVA_TRUSTED_PROXIES.Split(new[] { ',', ';', ' ' }, StringSplitOptions.RemoveEmptyEntries))
                {
                    if (entry.Contains('/') && System.Net.IPNetwork.TryParse(entry, out var network))
                        forwardedHeadersOptions.KnownIPNetworks.Add(network);
                    else if (!entry.Contains('/') && System.Net.IPAddress.TryParse(entry, out var address))
                        forwardedHeadersOptions.KnownProxies.Add(address);
                    else
                        _newLog.LogWarning($"AYANOVA_TRUSTED_PROXIES entry \"{entry}\" is not an IP address or network and is ignored");
                }
                //follow a chain of trusted proxies (e.g. a load balancer behind another proxy) back to the first address that is not trusted
                forwardedHeadersOptions.ForwardLimit = null;
            }
            app.UseForwardedHeaders(forwardedHeadersOptions);

            //case 4732: X-Forwarded-For from a proxy that is not trusted is ignored (no X-Original-For is set), so every user appears to come
            //from the proxy's address and shares one failed login limit; say so once in the log
            app.Use(async (context, next) =>
            {
                if (_untrustedProxyWarned == 0
                    && context.Request.Headers.ContainsKey("X-Forwarded-For")
                    && !context.Request.Headers.ContainsKey("X-Original-For")
                    && System.Threading.Interlocked.Exchange(ref _untrustedProxyWarned, 1) == 0)
                    _newLog.LogWarning($"A request came through a reverse proxy at {(context.Connection.RemoteIpAddress.IsIPv4MappedToIPv6 ? context.Connection.RemoteIpAddress.MapToIPv4() : context.Connection.RemoteIpAddress)} that is not trusted, so users connecting through it all appear to come from that address and share one limit on failed login attempts. If this is your reverse proxy, add its address to the AYANOVA_TRUSTED_PROXIES setting");
                await next.Invoke();
            });


            //If want to add a header to all (or filtered in here) requests
            // app.Use(async (context, next) =>
            // {
            //     context.Response.Headers.Add("X-Developed-By", "Ground Zero Tech-Works inc.");
            //     await next.Invoke();
            // });

            //case 4727 (F35): safe headers on every response, so self-hosted installs get them too (hosted droplets
            //also get them from nginx: same values, so the duplicates agree)
            app.Use(async (context, next) =>
            {
                context.Response.Headers["X-Content-Type-Options"] = "nosniff";
                context.Response.Headers["X-Frame-Options"] = "SAMEORIGIN";
                context.Response.Headers["Referrer-Policy"] = "strict-origin";
                await next.Invoke();
            });


            #region STATIC FILES
            _newLog.LogDebug("Static files");
            app.UseDefaultFiles();
            app.UseStaticFiles();
            //Might need the following if the page doesn't update in the client properly
            //however the vue build process will automatically uniquify each build file names so maybe not required
            // app.UseStaticFiles(new StaticFileOptions
            // {
            //     OnPrepareResponse = context =>
            //     {
            //         if (context.File.Name == "kindex.html")
            //         {
            //             context.Context.Response.Headers.Add("Cache-Control", "no-cache, no-store");
            //             context.Context.Response.Headers.Add("Expires", "-1");
            //         }
            //     }
            // });
            #endregion

            _newLog.LogDebug("Routing pipeline");
            app.UseRouting();//this wasn't here for 2.2 but added for 3.0, needs to come before the stuff after 

            _newLog.LogDebug("CORS pipeline");
            app.UseCors("CorsPolicy");


            #region AUTH / ROLES CUSTOM MIDDLEWARE
            _newLog.LogDebug("Authentication pipeline");
            //Use authentication middleware
            app.UseAuthentication();

            _newLog.LogDebug("Authorization pipeline");
            app.UseAuthorization();


            //Custom middleware to ensure token still valid and to
            //get user roles and put them into the request so 
            //they can be authorized in routes.
            app.Use(async (context, next) =>
            {
                if (!context.User.Identity.IsAuthenticated)
                {
                    context.Request.HttpContext.Items["AY_ROLES"] = 0;
                    await next.Invoke();
                }
                else
                {
                    //Get user ID from claims
                    long userId = Convert.ToInt64(context.User.FindFirst(c => c.Type == "id").Value);

                    //Get JWT
                    string JWT = string.Empty;
                    var AuthHeaders = context.Request.Headers[Microsoft.Net.Http.Headers.HeaderNames.Authorization];
                    foreach (String s in AuthHeaders)
                    {
                        if (s.ToLowerInvariant().Contains("bearer"))
                        {
                            JWT = s.Split(' ')[1];
                            break;
                        }
                    }

                    //Get the database context
                    var ct = context.RequestServices.GetService<AyContext>();

                    //get the user record
                    var u = await ct.User.AsNoTracking().Where(a => a.Id == userId).Select(m => new { roles = m.Roles, name = m.Name, m.UserType, id = m.Id, translationId = m.UserOptions.TranslationId, currentAuthToken = m.CurrentAuthToken, m.Active, m.AllowLogin, m.Login, m.Salt, m.Password }).FirstAsync();
                    //case 4713: a customer contact acts only with the customer roles, whatever roles are stored
                    context.Request.HttpContext.Items["AY_ROLES"] = UserBiz.EffectiveRoles(u.UserType, u.roles);
                    context.Request.HttpContext.Items["AY_USERNAME"] = u.name;
                    context.Request.HttpContext.Items["AY_USER_ID"] = u.id;
                    context.Request.HttpContext.Items["AY_TRANSLATION_ID"] = u.translationId;
                    context.Request.HttpContext.Items["AY_USER_TYPE"] = u.UserType;

                    var currentAuthToken = u.currentAuthToken;

                    //turned out didn't need this for v8 migrate so far, but keeping in case it turns out to be handy down the road
                    // //Is import mode header set?
                    // if (context.Request.Headers.ContainsKey("X-AY-Import-Mode"))
                    //     context.Request.HttpContext.Items["AY_IMPORT_MODE"] = true;


                    //CHECK JWT 
                    //case 4725: the login and prelogin routes are matched exactly (was any path ending in "/auth" or "notify/hello"),
                    //and a user who is inactive or may not log in is refused like a replaced token
                    var routeKey = AyaNova.Api.ControllerHelpers.CustomerContactRoutes.RouteKey(context);
                    if (
                        routeKey != "POST auth" &&
                        routeKey != "GET notify/hello" &&
                        (u.currentAuthToken != JWT || !u.Active || !u.AllowLogin)
                    )//except "/api/v8/auth" and prelogin notify/hello routes so user can login
                    {

                        //It may be a local report render request from a job
                        bool bLocalReportRenderRequest = false;
                        if (context.Request.Host.Host == "127.0.0.1")
                        {
                            //check if token has j value set
                            if (context.Request.IsLocal())
                            {
                                //if this is a generate report internal token this will be the override language set
                                //and it's existance is enough to indicate it was set internally by this server
                                var rpl = context.User.Claims.FirstOrDefault(c => c.Type == "rpl");
                                if (rpl != null)
                                {
                                    //it's a local request, the jwt already passed earlier so we issued it and it has the correct claim set so allow it
                                    bLocalReportRenderRequest = true;
                                    //set override language
                                    context.Request.HttpContext.Items["AY_TRANSLATION_ID"] = Convert.ToInt64(rpl.Value);
                                    await next.Invoke();
                                }
                            }
                        }
                        if (!bLocalReportRenderRequest)
                        {
                            context.Response.StatusCode = 401;
                            //case 4586
                            context.Response.Headers.Append("X-AyaNova-Authorization-Error", "E2004 - Authorization token replaced by more recent login");
                            await context.Response.WriteAsync("E2004 - Authorization token replaced by more recent login");
                        }
                    }
                    else
                    {
                        //case 4712: customer contacts may only call the routes the customer screens use
                        //case 4725: with a purchased licence, a SuperUser with the default password may only change it
                        if (((u.UserType == UserType.Customer || u.UserType == UserType.HeadOffice) && !AyaNova.Api.ControllerHelpers.CustomerContactRoutes.IsAllowed(context)) ||
                            (u.id == 1 && AyaNova.Core.License.PurchasedLicense() && UserBiz.IsDefaultSuperUserCreds(u.Login, u.Salt, u.Password) &&
                             !AyaNova.Api.ControllerHelpers.SuperUserDefaultPasswordRoutes.IsAllowed(context)))
                        {
                            context.Response.StatusCode = 403;
                            context.Response.ContentType = "application/json";
                            await context.Response.WriteAsync(Newtonsoft.Json.JsonConvert.SerializeObject(new AyaNova.Api.ControllerHelpers.ApiNotAuthorizedResponse(),
                                new Newtonsoft.Json.JsonSerializerSettings
                                {
                                    ContractResolver = new Newtonsoft.Json.Serialization.CamelCasePropertyNamesContractResolver(),
                                    NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore
                                }));
                            return;
                        }
                        await next.Invoke();
                    }
                }

            });

            #endregion


            _newLog.LogDebug("Endpoints pipeline");
            app.UseEndpoints(endpoints =>
            {
                endpoints.MapHealthChecks("/health");
                endpoints.MapControllers();
            });

            #region SWAGGER

            _newLog.LogDebug("API explorer pipeline");
            // Enable middleware to serve generated Swagger as a JSON endpoint.          
            app.UseSwagger();
            app.UseSwaggerUI(
                   options =>
                   {
                       foreach (var description in provider.ApiVersionDescriptions)
                       {
                           options.SwaggerEndpoint(
                               $"/swagger/{description.GroupName}/swagger.json",
                               description.GroupName.ToUpperInvariant());
                       }
                       options.DocExpansion(Swashbuckle.AspNetCore.SwaggerUI.DocExpansion.None);
                       options.DefaultModelsExpandDepth(-1);//This is meant to hide the Models section that would appear at the bottom of the swagger ui showing *all* models from the api
                       options.DocumentTitle = "AyaNova API explorer";
                       options.RoutePrefix = "api-docs";
                   });


            #endregion swagger




            if (ServerBootConfig.AYANOVA_PERMANENTLY_ERASE_DATABASE)
            {
                _newLog.LogWarning("AYANOVA_PERMANENTLY_ERASE_DATABASE has been set - deleting and recreating database");
                Util.DbUtil.DropAndRecreateDbAsync(_newLog).Wait();
                AySchema.CheckAndUpdateAsync(dbContext, _newLog).Wait();
            }



            var dbServerVersionInfo = DbUtil.DBServerVersion(dbContext);
            var dbServerRunTimeParameters = DbUtil.DBServerRunTimeParameters(dbContext);
            //Log server version
            _newLog.LogInformation("Database server version - {0}", dbServerVersionInfo);

            //db server extended parameters
            _newLog.LogTrace($"Database server runtime parameters{Environment.NewLine}{string.Join(Environment.NewLine, dbServerRunTimeParameters)}");

            //log each item individually from runtime parameters
            ServerBootConfig.DBSERVER_DIAGNOSTIC_INFO.Add("DB SERVER", dbServerVersionInfo);
            foreach (var p in dbServerRunTimeParameters)
                ServerBootConfig.DBSERVER_DIAGNOSTIC_INFO.Add(p.Key, p.Value);



            //case 4170
            if (ServerBootConfig.AYANOVA_REMOVE_LICENSE_FROM_DB)
            {
                _newLog.LogWarning("AYANOVA_REMOVE_LICENSE_FROM_DB has been set - removing license from database");
                Util.DbUtil.RemoveLicenseFromDB(_newLog);
            }


            //BOOT safety check to prevent schema update if user has installed version of AyaNova they are not licensed for           
            try
            {
                //this will either return without throwing which is ok or will throw and may or may not indicate need to be prevented from further boot
                AyaNova.Core.License.BootSafetyCheck(dbContext, _newLog);
            }
            catch (Exception ex)
            {
                //Only re-throw if it's a 1020 error with the text defined in LICENSE_MISMATCH_TO_BUILD_ERROR included, any other error allow it to keep doing it's thing below before the second license init
                //our exception is buried inside multiple inner exceptions but it's the innermost so drill down into it                    
                while (ex.InnerException != null)
                    ex = ex.InnerException;
                if (ex.Message.Contains("1020") && ex.Message.Contains(AyaNova.Core.License.LICENSE_MISMATCH_TO_BUILD_ERROR))//DO NOT CHANGE THIS MAGIC STRING KEY
                {
                    throw new Exception("AyaNova did not start to protect the integrity of your data, see the console and / or error log for details");
                }
            }

            //Check schema
            _newLog.LogDebug("DB schema check");
            AySchema.CheckAndUpdateAsync(dbContext, _newLog).Wait();


            //Check database integrity
            _newLog.LogDebug("DB integrity check");
            DbUtil.CheckFingerPrintAsync(AySchema.EXPECTED_COLUMN_COUNT,
                                         AySchema.EXPECTED_INDEX_COUNT,
                                        //case 4640 AySchema.EXPECTED_CHECK_CONSTRAINTS,
                                         AySchema.EXPECTED_FOREIGN_KEY_CONSTRAINTS,
                                         AySchema.EXPECTED_VIEWS,
                                         AySchema.EXPECTED_ROUTINES,
                                         _newLog).Wait();

            //Initialize license   
            AyaNova.Core.License.InitializeAsync(apiServerState, dbContext, _newLog).Wait();

            //Set static global biz settings
            _newLog.LogDebug("Global settings");
            ServerGlobalBizSettings.Initialize(null, dbContext);

            _newLog.LogDebug("Ops settings");
            ServerGlobalOpsSettingsCache.Initialize(dbContext);

            //Ensure translations are present, not missing any keys and that there is a server default translation that exists          
            TranslationBiz lb = new TranslationBiz(dbContext, 1, ServerBootConfig.AYANOVA_DEFAULT_TRANSLATION_ID, AuthorizationRoles.OpsAdmin);
            lb.ValidateTranslationsAsync().Wait();


            //SPA FALLBACK ROUTE                 
            app.Use(async (context, next) =>
            {
                //to support html5 pushstate routing in spa
                //this ensures that a refresh at the client will not 404 but rather force back to the index.html app page and then handled internally by the client 
                await next();
                if (!context.Response.HasStarted && !context.Request.Path.Value.StartsWith("/api") && context.Request.Path.Value != "/docs" && context.Request.Path.Value != "/cust" && context.Response.StatusCode == 404 && !Path.HasExtension(context.Request.Path.Value))
                {
                    context.Request.Path = "/index.html";
                    context.Response.StatusCode = 200;
                    context.Response.ContentType = "text/html";
                    await context.Response.SendFileAsync(Path.Combine(env.WebRootPath, "index.html"));
                }
            });


            //Log schema version into server log (would only otherwise log if schema was out of date)
            _newLog.LogInformation($"DB Schema version - {AyaNova.Util.AySchema.currentSchema}");
            ServerBootConfig.BOOT_DIAGNOSTIC_INFO.Add("DB Schema version", AyaNova.Util.AySchema.currentSchema.ToString());


            //Log the active user counts so it's in the log record
            //I'm doing all three even though some not relevant in certain scenarios in case of user switching between subscription/perpetual as another source of information
            //_newLog.LogInformation($"Active tech Users - {UserBiz.ActiveTechUserCountAsync().Result}");
            _newLog.LogInformation($"Active Customer contact Users - {UserBiz.ActiveCustomerContactUserCountAsync().Result}");
            _newLog.LogInformation($"Active internal staff Users - {UserBiz.ActiveInternalUserCountAsync().Result}");

            //Log the license info so it's on the record
            _newLog.LogInformation($"License - [{AyaNova.Core.License.LicenseInfoLogFormat}]");


            //Check for SuperUser password override
            if (!string.IsNullOrWhiteSpace(ServerBootConfig.AYANOVA_SET_SUPERUSER_PW))
            {
                _newLog.LogWarning($"### AYANOVA_SET_SUPERUSER_PW IS PRESENT - RESETTING SUPERUSER PASSWORD NOW... ###");
                AyaNova.Biz.UserBiz.ResetSuperUserPassword();
                _newLog.LogWarning($"### AYANOVA_SET_SUPERUSER_PW HAS BEEN USED TO RESET SUPER USER PASSWORD YOU CAN REMOVE THIS SETTING NOW ###");
            }

            //Boot lock for generator
            ServerGlobalOpsSettingsCache.BOOTING = false;

            //Open up the server for visitors
            _newLog.LogDebug("Setting server state open");
            apiServerState.SetOpen();

            //final startup log
            _newLog.LogInformation("Boot complete - server open");

            //flag in console that server is open
            Console.WriteLine("-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-");
            Console.WriteLine("BOOT: COMPLETED - SERVER OPEN");
            Console.WriteLine($"AYANOVA_USE_URLS setting: \"{ServerBootConfig.AYANOVA_USE_URLS}\"");
            Console.WriteLine("Controlled shutdown: AyaNova APP -> \"Operations\" -> \"ServerState\" -> \"Shut down server\" from menu");
            Console.WriteLine("Forced shutdown: Ctrl+C keyboard shortcut");
            Console.WriteLine("-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-");

        }


        #region Swagger and API Versioning utilities

        static string XmlCommentsFilePath
        {
            get
            {
                var basePath = AppContext.BaseDirectory;
                var fileName = typeof(Startup).GetTypeInfo().Assembly.GetName().Name + ".xml";
                return Path.Combine(basePath, fileName);
            }
        }


        #endregion
    }
}