admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Browse Source
This commit is contained in:
John Cardinal
2018-10-03 18:10:45 +00:00
parent 896521fbed
commit aee2234092
6 changed files with 47 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
devdocs/specs/core-performance.txt
View File

@@ -67,7 +67,7 @@ SELECT indexrelname,cast(idx_tup_read AS numeric) / idx_scan AS avg_tuples,idx_s
WORK IN PROGRESS:
Search result list:
Search result list NAME FETCHER in :
//Before attempt to optimize name fetcher (unknown number of results)
//22548, 21187, 20462, 22336, 20094 - AVG = 21325
@@ -92,3 +92,13 @@ Now I'm going to try it with the index put back in and data regenerated
Now fresh test but without index being crated
### 14244 results with index in place (after a restart of server, and bypassing EF entirely with a direct query OPTIMIZED TO REUSE CONNECTION): 14270 results - 13176, 12688, 13179, 12994, 12272 AVG: 12,861 = .90 per result
index put back in and data regenerated
### 14255 results with index in place (after a restart of server and bypassing EF entirely with a direct query OPTIMIZED TO REUSE CONNECTION): 12461, 12040, 11171, 11141, 11214 AVG: 11605 = .81 per result
OK, this tells me that it's faster with the index in place and intuitively that just makes sense.
Also verified it's actually using the index scan instead of table scan.
I'm going to enact a policy to index id,name in all objects that have many columns, if they only have a name and id and not much else then there seems little benefit
### results ("final" id,name indexes on user table and widget table, freshly generated data), 14202 RESULTS: 13295, 14502, 11774, 12521, 12101, 13169 AVG: 12,893 = 1.15
Ok, it just makes logical sense to keep the indexes even if slightly slower, I can revisit this later, the difference is miniscule. I suspect with a bigger database there would definitely be better peformance.

1
devdocs/todo.txt
View File

@@ -50,6 +50,7 @@ Current unbounded search for "et*" in huge dataset takes avg 21 seconds to proce
- see if any other callers to name fetcher are in tight loops and could benefit from using the new Direct version
- Update all the other routes to include search indexing (attachments, tags etc, anything with text in it)
- Schema: clean up all the LOOKAT items and verify the indexes are being used
- EventLogProcessor.AddEntry: CHANGE this to save the context itself and then change all callers to handle that (remove save)
- I originally didn't have the save in there because I thought subsequent code might all share in the single context save,

2
server/AyaNova/Startup.cs
View File

@@ -363,7 +363,7 @@ namespace AyaNova
// ******************** TESTING WIPE DB *****************************
//
//Set this to true to wipe the db and reinstall a trial license and re-seed the data
var TESTING_REFRESH_DB = true;
var TESTING_REFRESH_DB = false;
#if (DEBUG)
//TESTING

15
server/AyaNova/biz/BizObjectNameFetcherDirect.cs
View File

@@ -13,21 +13,8 @@ using AyaNova.Models;
namespace AyaNova.Biz
{
/*
using (var command = ct.Database.GetDbConnection().CreateCommand())
{
command.CommandText = $"SELECT m.name FROM awidget AS m WHERE m.id = {id} LIMIT 1";
ct.Database.OpenConnection();
using (var dr = command.ExecuteReader())
{
// do something with result
return dr.Read() ? dr.GetString(0) : "UNKNOWN";
}
}
*/
//Turn a type and ID into a displayable name
//this version uses a direct DataReader for performance in tight loops (search)
internal static class BizObjectNameFetcherDirect
{

27
server/AyaNova/biz/Search.cs
View File

@@ -375,32 +375,7 @@ namespace AyaNova.Biz
watch.Stop();//###################### PROFILING
var TimeToBuildSearchResultReturnList = watch.ElapsedMilliseconds;//###################### PROFILING
//Before attempt to optimize name fetcher
//22548, 21187, 20462, 22336, 20094 - AVG = 21325
/*
explain analyze SELECT m.name
FROM awidget AS m
WHERE m.id = 12989
LIMIT 1
"Limit (cost=0.29..8.30 rows=1 width=27) (actual time=0.079..0.080 rows=1 loops=1)"
" -> Index Scan using awidget_pkey on awidget m (cost=0.29..8.30 rows=1 width=27) (actual time=0.077..0.077 rows=1 loops=1)"
" Index Cond: (id = 12989)"
"Planning time: 0.098 ms"
"Execution time: 0.102 ms"
//All index data
select * from pg_stat_user_indexes
*/
return ResultList;
}

53
server/AyaNova/util/AySchema.cs
View File

@@ -23,12 +23,26 @@ namespace AyaNova.Util
private const int DESIRED_SCHEMA_LEVEL = 9;
internal const long EXPECTED_COLUMN_COUNT = 99;
internal const long EXPECTED_INDEX_COUNT = 22;
internal const long EXPECTED_INDEX_COUNT = 23;
//!!!!WARNING: BE SURE TO UPDATE THE DbUtil::PrepareDatabaseForSeeding WHEN NEW TABLES ADDED!!!!
/////////////////////////////////////////////////////////////////
/*
MAXIMUM POSTGRES OBJECT NAME LENGTH: 63 CHARACTERS
HOW TO INDEX
AyaNova does a lot of name fetching so any tables that contain a lot of columns in addition to the name will benefit from a compound index on (id,name)
Other indexes should be created with care and after a huge load and integration test periodically look for unused indexes and see how they are performing
see core-performance.txt for the relevant queries to view this info
*/
static int startingSchema = -1;
public static int currentSchema = -1;
@@ -125,37 +139,30 @@ namespace AyaNova.Util
exec("CREATE TABLE aevent (id BIGSERIAL PRIMARY KEY, created timestamp not null, ownerid bigint not null," +
"ayid bigint not null, aytype integer not null, ayevent integer not null, textra varchar(255))");
//LOOKAT: do I *really* need these or do they bloat unnecessarily? Need to test with big dataset
//index for quick searching
// exec("CREATE INDEX ayid_idx ON aevent (ayid);");
// exec("CREATE INDEX aytype_idx ON aevent (aytype);");
//Add the search key and dictionary tables
//TODO: Indexes determined through load testing and experimentation
//Too many indexes or unnecessary ones would be bad because this table is hit on every save / update of an object and would slow those ops
//too little is bad if search takes a dogs age to find anything
//SEARCH TABLES
exec("CREATE TABLE asearchdictionary (id BIGSERIAL PRIMARY KEY, word varchar(255) not null)");
exec("CREATE UNIQUE INDEX searchdictword_idx ON asearchdictionary (word);");
exec("CREATE UNIQUE INDEX asearchdictionary_word_idx ON asearchdictionary (word);");
exec("CREATE TABLE asearchkey (id BIGSERIAL PRIMARY KEY, wordid bigint not null REFERENCES asearchdictionary (id), objectid bigint not null, objecttype integer not null, inname bool not null)");
//create locale text tables
exec("CREATE TABLE alocale (id BIGSERIAL PRIMARY KEY, ownerid bigint not null, name varchar(255) not null, stock bool, cjkindex bool default false)");
exec("CREATE UNIQUE INDEX localename_idx ON alocale (name)");
exec("CREATE UNIQUE INDEX alocale_name_idx ON alocale (name)");
exec("CREATE TABLE alocaleitem (id BIGSERIAL PRIMARY KEY, localeid bigint not null REFERENCES alocale (id), key text not null, display text not null)");
exec("CREATE INDEX localeitemlid_key_idx ON alocaleitem (localeid,key)");
exec("CREATE INDEX alocaleitem_localeid_key_idx ON alocaleitem (localeid,key)");
//Load the default LOCALES
AyaNova.Biz.PrimeData.PrimeLocales(ct);
//Add user table
exec("CREATE TABLE auser (id BIGSERIAL PRIMARY KEY, ownerid bigint not null, active bool not null, name varchar(255) not null, " +
"login text not null, password text not null, salt text not null, roles integer not null, localeid bigint not null REFERENCES alocale (id), " +
"dlkey text, dlkeyexpire timestamp, usertype integer not null, employeenumber varchar(255), notes text, clientid bigint, headofficeid bigint, subvendorid bigint)");
//Index for name fetching
exec("CREATE UNIQUE INDEX auser_name_id_idx ON auser (id, name);");
//Add user options table
exec("CREATE TABLE auseroptions (id BIGSERIAL PRIMARY KEY, ownerid bigint not null, " +
"userid bigint not null, timezoneoffset decimal(19,5) not null default 0, emailaddress text, uicolor int not null default 0)");
@@ -192,9 +199,8 @@ namespace AyaNova.Util
exec("CREATE TABLE awidget (id BIGSERIAL PRIMARY KEY, ownerid bigint not null, name varchar(255) not null, " +
"startdate timestamp, enddate timestamp, dollaramount decimal(19,5), active bool, roles int4, notes text)");
//PERF TESTING INDEX ############################
//Weirdly, it's always slower when it uses the index?? memory issue maybe?
exec("CREATE UNIQUE INDEX widget_idx_name_id ON public.awidget USING btree (id, name COLLATE pg_catalog.\"default\") TABLESPACE pg_default;");
//Compound index for name fetching
exec("CREATE UNIQUE INDEX awidget_name_id_idx ON awidget (id, name);");
setSchemaLevel(++currentSchema);
}
@@ -211,7 +217,8 @@ namespace AyaNova.Util
"storedfilename text not null, displayfilename text not null, contenttype text, notes text)");
//index required for ops that need to check if file already in db (delete, count refs etc)
exec("CREATE INDEX storedfilename_idx ON afileattachment (storedfilename);");
//LOOKAT: isn't this useless without the ID as well or is that not fetched?
exec("CREATE INDEX afileattachment_storedfilename_idx ON afileattachment (storedfilename);");
setSchemaLevel(++currentSchema);
}
@@ -224,7 +231,10 @@ namespace AyaNova.Util
LogUpdateMessage(log);
// LOOKAT: Should taggroupmap have an index that enforces no taggroup can have the same tag more than once? Same for objects being tagged?
exec("CREATE TABLE atag (id BIGSERIAL PRIMARY KEY, ownerid bigint not null, name varchar(255) not null)");
exec("CREATE UNIQUE INDEX tagname_idx ON atag (name);");
//LOOKAT: isn't this useless without the ID? Need to see if it's being used after unit testing
exec("CREATE UNIQUE INDEX atag_name_idx ON atag (name);");
exec("CREATE TABLE atagmap (id BIGSERIAL PRIMARY KEY, ownerid bigint not null," +
"tagid bigint not null REFERENCES atag (id), tagtoobjectid bigint not null, tagtoobjecttype integer not null)");
@@ -253,6 +263,7 @@ namespace AyaNova.Util
//////////////////////////////////////////////////
//LICENSE table new columns
//LOOKAT: DO I need this anymore???
//answer: no because it relates to ops stuff in other tables and logging, not to the license itself (except maybe dbid?)
if (currentSchema < 8)
{
LogUpdateMessage(log);