From aee2234092978d62f7a38d6a2c574d7ff2a63c87 Mon Sep 17 00:00:00 2001 From: John Cardinal Date: Wed, 3 Oct 2018 18:10:45 +0000 Subject: [PATCH] --- devdocs/specs/core-performance.txt | 12 ++++- devdocs/todo.txt | 1 + server/AyaNova/Startup.cs | 2 +- .../AyaNova/biz/BizObjectNameFetcherDirect.cs | 15 +----- server/AyaNova/biz/Search.cs | 27 +--------- server/AyaNova/util/AySchema.cs | 53 +++++++++++-------- 6 files changed, 47 insertions(+), 63 deletions(-) diff --git a/devdocs/specs/core-performance.txt b/devdocs/specs/core-performance.txt index 21bdabe3..9e40f1ac 100644 --- a/devdocs/specs/core-performance.txt +++ b/devdocs/specs/core-performance.txt @@ -67,7 +67,7 @@ SELECT indexrelname,cast(idx_tup_read AS numeric) / idx_scan AS avg_tuples,idx_s WORK IN PROGRESS: -Search result list: +Search result list NAME FETCHER in : //Before attempt to optimize name fetcher (unknown number of results) //22548, 21187, 20462, 22336, 20094 - AVG = 21325 @@ -92,3 +92,13 @@ Now I'm going to try it with the index put back in and data regenerated Now fresh test but without index being crated ### 14244 results with index in place (after a restart of server, and bypassing EF entirely with a direct query OPTIMIZED TO REUSE CONNECTION): 14270 results - 13176, 12688, 13179, 12994, 12272 AVG: 12,861 = .90 per result +index put back in and data regenerated +### 14255 results with index in place (after a restart of server and bypassing EF entirely with a direct query OPTIMIZED TO REUSE CONNECTION): 12461, 12040, 11171, 11141, 11214 AVG: 11605 = .81 per result +OK, this tells me that it's faster with the index in place and intuitively that just makes sense. +Also verified it's actually using the index scan instead of table scan. +I'm going to enact a policy to index id,name in all objects that have many columns, if they only have a name and id and not much else then there seems little benefit + +### results ("final" id,name indexes on user table and widget table, freshly generated data), 14202 RESULTS: 13295, 14502, 11774, 12521, 12101, 13169 AVG: 12,893 = 1.15 + +Ok, it just makes logical sense to keep the indexes even if slightly slower, I can revisit this later, the difference is miniscule. I suspect with a bigger database there would definitely be better peformance. + diff --git a/devdocs/todo.txt b/devdocs/todo.txt index f0e42b07..ae4f0b54 100644 --- a/devdocs/todo.txt +++ b/devdocs/todo.txt @@ -50,6 +50,7 @@ Current unbounded search for "et*" in huge dataset takes avg 21 seconds to proce - see if any other callers to name fetcher are in tight loops and could benefit from using the new Direct version - Update all the other routes to include search indexing (attachments, tags etc, anything with text in it) + - Schema: clean up all the LOOKAT items and verify the indexes are being used - EventLogProcessor.AddEntry: CHANGE this to save the context itself and then change all callers to handle that (remove save) - I originally didn't have the save in there because I thought subsequent code might all share in the single context save, diff --git a/server/AyaNova/Startup.cs b/server/AyaNova/Startup.cs index 935ce0e5..0e131fea 100644 --- a/server/AyaNova/Startup.cs +++ b/server/AyaNova/Startup.cs @@ -363,7 +363,7 @@ namespace AyaNova // ******************** TESTING WIPE DB ***************************** // //Set this to true to wipe the db and reinstall a trial license and re-seed the data - var TESTING_REFRESH_DB = true; + var TESTING_REFRESH_DB = false; #if (DEBUG) //TESTING diff --git a/server/AyaNova/biz/BizObjectNameFetcherDirect.cs b/server/AyaNova/biz/BizObjectNameFetcherDirect.cs index 9206e655..dd37e335 100644 --- a/server/AyaNova/biz/BizObjectNameFetcherDirect.cs +++ b/server/AyaNova/biz/BizObjectNameFetcherDirect.cs @@ -13,21 +13,8 @@ using AyaNova.Models; namespace AyaNova.Biz { - /* - - using (var command = ct.Database.GetDbConnection().CreateCommand()) - { - command.CommandText = $"SELECT m.name FROM awidget AS m WHERE m.id = {id} LIMIT 1"; - ct.Database.OpenConnection(); - using (var dr = command.ExecuteReader()) - { - - // do something with result - return dr.Read() ? dr.GetString(0) : "UNKNOWN"; - } - } - */ //Turn a type and ID into a displayable name + //this version uses a direct DataReader for performance in tight loops (search) internal static class BizObjectNameFetcherDirect { diff --git a/server/AyaNova/biz/Search.cs b/server/AyaNova/biz/Search.cs index 101c4544..aecccf44 100644 --- a/server/AyaNova/biz/Search.cs +++ b/server/AyaNova/biz/Search.cs @@ -375,32 +375,7 @@ namespace AyaNova.Biz watch.Stop();//###################### PROFILING var TimeToBuildSearchResultReturnList = watch.ElapsedMilliseconds;//###################### PROFILING - //Before attempt to optimize name fetcher - //22548, 21187, 20462, 22336, 20094 - AVG = 21325 - - /* - explain analyze SELECT m.name -FROM awidget AS m -WHERE m.id = 12989 -LIMIT 1 - - -"Limit (cost=0.29..8.30 rows=1 width=27) (actual time=0.079..0.080 rows=1 loops=1)" -" -> Index Scan using awidget_pkey on awidget m (cost=0.29..8.30 rows=1 width=27) (actual time=0.077..0.077 rows=1 loops=1)" -" Index Cond: (id = 12989)" -"Planning time: 0.098 ms" -"Execution time: 0.102 ms" - - -//All index data -select * from pg_stat_user_indexes - - - - - - - */ + return ResultList; } diff --git a/server/AyaNova/util/AySchema.cs b/server/AyaNova/util/AySchema.cs index 7a99a05e..23150e55 100644 --- a/server/AyaNova/util/AySchema.cs +++ b/server/AyaNova/util/AySchema.cs @@ -23,12 +23,26 @@ namespace AyaNova.Util private const int DESIRED_SCHEMA_LEVEL = 9; internal const long EXPECTED_COLUMN_COUNT = 99; - internal const long EXPECTED_INDEX_COUNT = 22; + internal const long EXPECTED_INDEX_COUNT = 23; + //!!!!WARNING: BE SURE TO UPDATE THE DbUtil::PrepareDatabaseForSeeding WHEN NEW TABLES ADDED!!!! ///////////////////////////////////////////////////////////////// + /* + + MAXIMUM POSTGRES OBJECT NAME LENGTH: 63 CHARACTERS + + HOW TO INDEX + + AyaNova does a lot of name fetching so any tables that contain a lot of columns in addition to the name will benefit from a compound index on (id,name) + + Other indexes should be created with care and after a huge load and integration test periodically look for unused indexes and see how they are performing + see core-performance.txt for the relevant queries to view this info + + */ + static int startingSchema = -1; public static int currentSchema = -1; @@ -125,37 +139,30 @@ namespace AyaNova.Util exec("CREATE TABLE aevent (id BIGSERIAL PRIMARY KEY, created timestamp not null, ownerid bigint not null," + "ayid bigint not null, aytype integer not null, ayevent integer not null, textra varchar(255))"); - //LOOKAT: do I *really* need these or do they bloat unnecessarily? Need to test with big dataset - //index for quick searching - // exec("CREATE INDEX ayid_idx ON aevent (ayid);"); - // exec("CREATE INDEX aytype_idx ON aevent (aytype);"); - - //Add the search key and dictionary tables - //TODO: Indexes determined through load testing and experimentation - //Too many indexes or unnecessary ones would be bad because this table is hit on every save / update of an object and would slow those ops - //too little is bad if search takes a dogs age to find anything - + //SEARCH TABLES exec("CREATE TABLE asearchdictionary (id BIGSERIAL PRIMARY KEY, word varchar(255) not null)"); - exec("CREATE UNIQUE INDEX searchdictword_idx ON asearchdictionary (word);"); + exec("CREATE UNIQUE INDEX asearchdictionary_word_idx ON asearchdictionary (word);"); exec("CREATE TABLE asearchkey (id BIGSERIAL PRIMARY KEY, wordid bigint not null REFERENCES asearchdictionary (id), objectid bigint not null, objecttype integer not null, inname bool not null)"); //create locale text tables exec("CREATE TABLE alocale (id BIGSERIAL PRIMARY KEY, ownerid bigint not null, name varchar(255) not null, stock bool, cjkindex bool default false)"); - exec("CREATE UNIQUE INDEX localename_idx ON alocale (name)"); + exec("CREATE UNIQUE INDEX alocale_name_idx ON alocale (name)"); + exec("CREATE TABLE alocaleitem (id BIGSERIAL PRIMARY KEY, localeid bigint not null REFERENCES alocale (id), key text not null, display text not null)"); - exec("CREATE INDEX localeitemlid_key_idx ON alocaleitem (localeid,key)"); + exec("CREATE INDEX alocaleitem_localeid_key_idx ON alocaleitem (localeid,key)"); //Load the default LOCALES AyaNova.Biz.PrimeData.PrimeLocales(ct); - - //Add user table exec("CREATE TABLE auser (id BIGSERIAL PRIMARY KEY, ownerid bigint not null, active bool not null, name varchar(255) not null, " + "login text not null, password text not null, salt text not null, roles integer not null, localeid bigint not null REFERENCES alocale (id), " + "dlkey text, dlkeyexpire timestamp, usertype integer not null, employeenumber varchar(255), notes text, clientid bigint, headofficeid bigint, subvendorid bigint)"); + //Index for name fetching + exec("CREATE UNIQUE INDEX auser_name_id_idx ON auser (id, name);"); + //Add user options table exec("CREATE TABLE auseroptions (id BIGSERIAL PRIMARY KEY, ownerid bigint not null, " + "userid bigint not null, timezoneoffset decimal(19,5) not null default 0, emailaddress text, uicolor int not null default 0)"); @@ -192,9 +199,8 @@ namespace AyaNova.Util exec("CREATE TABLE awidget (id BIGSERIAL PRIMARY KEY, ownerid bigint not null, name varchar(255) not null, " + "startdate timestamp, enddate timestamp, dollaramount decimal(19,5), active bool, roles int4, notes text)"); - //PERF TESTING INDEX ############################ - //Weirdly, it's always slower when it uses the index?? memory issue maybe? - exec("CREATE UNIQUE INDEX widget_idx_name_id ON public.awidget USING btree (id, name COLLATE pg_catalog.\"default\") TABLESPACE pg_default;"); + //Compound index for name fetching + exec("CREATE UNIQUE INDEX awidget_name_id_idx ON awidget (id, name);"); setSchemaLevel(++currentSchema); } @@ -211,7 +217,8 @@ namespace AyaNova.Util "storedfilename text not null, displayfilename text not null, contenttype text, notes text)"); //index required for ops that need to check if file already in db (delete, count refs etc) - exec("CREATE INDEX storedfilename_idx ON afileattachment (storedfilename);"); + //LOOKAT: isn't this useless without the ID as well or is that not fetched? + exec("CREATE INDEX afileattachment_storedfilename_idx ON afileattachment (storedfilename);"); setSchemaLevel(++currentSchema); } @@ -224,7 +231,10 @@ namespace AyaNova.Util LogUpdateMessage(log); // LOOKAT: Should taggroupmap have an index that enforces no taggroup can have the same tag more than once? Same for objects being tagged? exec("CREATE TABLE atag (id BIGSERIAL PRIMARY KEY, ownerid bigint not null, name varchar(255) not null)"); - exec("CREATE UNIQUE INDEX tagname_idx ON atag (name);"); + + //LOOKAT: isn't this useless without the ID? Need to see if it's being used after unit testing + exec("CREATE UNIQUE INDEX atag_name_idx ON atag (name);"); + exec("CREATE TABLE atagmap (id BIGSERIAL PRIMARY KEY, ownerid bigint not null," + "tagid bigint not null REFERENCES atag (id), tagtoobjectid bigint not null, tagtoobjecttype integer not null)"); @@ -253,6 +263,7 @@ namespace AyaNova.Util ////////////////////////////////////////////////// //LICENSE table new columns //LOOKAT: DO I need this anymore??? + //answer: no because it relates to ops stuff in other tables and logging, not to the license itself (except maybe dbid?) if (currentSchema < 8) { LogUpdateMessage(log);