finished this round of schema improvements
This commit is contained in:
2
.vscode/launch.json
vendored
2
.vscode/launch.json
vendored
@@ -51,7 +51,7 @@
|
||||
"AYANOVA_FOLDER_BACKUP_FILES": "c:\\temp\\RavenTestData\\backupfiles",
|
||||
"AYANOVA_METRICS_USE_INFLUXDB": "false",
|
||||
"AYANOVA_SERVER_TEST_MODE":"true",
|
||||
"AYANOVA_SERVER_TEST_MODE_SEEDLEVEL":"huge",
|
||||
"AYANOVA_SERVER_TEST_MODE_SEEDLEVEL":"small",
|
||||
"AYANOVA_SERVER_TEST_MODE_TZ_OFFSET":"-7",
|
||||
"AYANOVA_BACKUP_PG_DUMP_PATH":"C:\\data\\code\\PostgreSQLPortable_12.0\\App\\PgSQL\\bin\\"
|
||||
|
||||
|
||||
@@ -8,6 +8,10 @@ http://okigiveup.net/what-postgresql-tells-you-about-its-performance/
|
||||
Useful queries to indicate how indexes are being used in postgresql
|
||||
=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
|
||||
|
||||
2020-05-21 THIS SUPERSEDES BELOW
|
||||
HOW TO FIND SHITTY INDEXES: https://gist.github.com/jberkus/6b1bcaf7724dfc2a54f3
|
||||
|
||||
==-=-=-
|
||||
OK: this is the best query to use to find non used indexes
|
||||
Run it and look for any indexes that are *not* primary keys (name ends in pkey, want ones that end in idx which are mine)
|
||||
that have zero in idx_scan, that means they are completely unused (if a lot of full test runs that excercise all routes and features have happened prior to checking)
|
||||
|
||||
@@ -1,65 +1,6 @@
|
||||
{"login": "manager","password": "l3tm3in"}
|
||||
|
||||
|
||||
todo: Search indexing performance improvement and exception avoidance (Search.cs 828)
|
||||
ON CONFLICT IDEA
|
||||
https://www.postgresql.org/docs/current/sql-insert.html#SQL-ON-CONFLICT
|
||||
Idea: do the insert manually with the clause "on conflict do nothing"
|
||||
if detect it hasn't inserted (conflict) trigger a fetch instead
|
||||
like what is being done now but won't have the exception to deal with!!
|
||||
var CtAdd.SearchDictionary.FromSqlRaw("insert into asearchdictionary (word) values('{0}') on conflict (word) do update set word=excluded.word returning id",KeyWord ).FirstOrDefaultAsync();
|
||||
|
||||
stored procedure?
|
||||
https://www.postgresqltutorial.com/plpgsql-loop-statements/
|
||||
-------
|
||||
CREATE OR REPLACE PROCEDURE public.aydosearchindex(wordlist text[], ayobjectid bigint, ayobjecttype integer, cleanfirst boolean)
|
||||
LANGUAGE 'plpgsql'
|
||||
|
||||
AS $BODY$DECLARE
|
||||
s text;
|
||||
wordid bigint;
|
||||
BEGIN
|
||||
IF ayobjectid=0 THEN
|
||||
RAISE EXCEPTION 'Bad object id --> %', ayobjectid;
|
||||
END IF;
|
||||
|
||||
IF ayobjecttype=0 THEN
|
||||
RAISE EXCEPTION 'Bad object type --> %', ayobjecttype;
|
||||
END IF;
|
||||
|
||||
|
||||
IF cleanfirst=true THEN
|
||||
delete from asearchkey where objectid=ayobjectid and objecttype=ayobjecttype;
|
||||
END IF;
|
||||
|
||||
FOREACH s IN ARRAY wordlist
|
||||
LOOP
|
||||
raise info 'processing word %',s;
|
||||
SELECT id INTO wordid FROM asearchdictionary WHERE word = s;
|
||||
raise info 'initial select found that word id is %', wordid;
|
||||
IF wordid = NULL THEN
|
||||
raise info 'since wordid was null inserting %...', wordid;
|
||||
insert into asearchdictionary (word) values(s) on conflict (word) do update set word=excluded.word returning id into wordid;
|
||||
raise info 'After insert new word returned word id %, inserting into searchkey', wordid;
|
||||
insert into asearchkey (wordid,objectid,objecttype) values(wordid,ayobjectid,ayobjecttype);
|
||||
ELSE
|
||||
raise info 'since we have initial word id from select inserting into search key the Word id %', wordid;
|
||||
insert into asearchkey (wordid,objectid,objecttype) values(wordid,ayobjectid,ayobjecttype);
|
||||
END IF;
|
||||
END LOOP;
|
||||
END;
|
||||
$BODY$;
|
||||
|
||||
------
|
||||
call aydosearchindex(ARRAY['sun','mon','tue','wed','thu','fri','sat'],3,3)
|
||||
-------
|
||||
|
||||
|
||||
|
||||
todo: Search confirm indexes are actually being used
|
||||
|
||||
|
||||
|
||||
todo: OPS notification created for failed jobs
|
||||
also maybe direct immediate email bypassing generator?
|
||||
Add backup fail to this will stub out for now
|
||||
|
||||
@@ -39,15 +39,98 @@ namespace AyaNova.Util
|
||||
AyaNova does a lot of name fetching so any tables that contain a lot of columns in addition to the name will benefit from a compound index on (id,name)
|
||||
|
||||
Other indexes should be created with care and after a huge load and integration test periodically look for unused indexes and see how they are performing
|
||||
HOW TO FIND SHITTY INDEXES: https://gist.github.com/jberkus/6b1bcaf7724dfc2a54f3
|
||||
see core-performance.txt for the relevant queries to view this info
|
||||
|
||||
***************************** WARNING: Be careful here, if a standard field is hideable and also it's DB SCHEMA is set to NON NULLABLE then the CLIENT end needs to set a default
|
||||
***************************** Otherwise the hidden field can't be set and the object can't be saved EVER
|
||||
|
||||
|
||||
HOW TO FIND UNUSED INDEXES: https://gist.github.com/jberkus/6b1bcaf7724dfc2a54f3
|
||||
|
||||
|
||||
*/
|
||||
#region unused index query
|
||||
/*
|
||||
COPY taken 2020-05-21 from link above "jerkus" :)
|
||||
|
||||
WITH table_scans as (
|
||||
SELECT relid,
|
||||
tables.idx_scan + tables.seq_scan as all_scans,
|
||||
( tables.n_tup_ins + tables.n_tup_upd + tables.n_tup_del ) as writes,
|
||||
pg_relation_size(relid) as table_size
|
||||
FROM pg_stat_user_tables as tables
|
||||
),
|
||||
all_writes as (
|
||||
SELECT sum(writes) as total_writes
|
||||
FROM table_scans
|
||||
),
|
||||
indexes as (
|
||||
SELECT idx_stat.relid, idx_stat.indexrelid,
|
||||
idx_stat.schemaname, idx_stat.relname as tablename,
|
||||
idx_stat.indexrelname as indexname,
|
||||
idx_stat.idx_scan,
|
||||
pg_relation_size(idx_stat.indexrelid) as index_bytes,
|
||||
indexdef ~* 'USING btree' AS idx_is_btree
|
||||
FROM pg_stat_user_indexes as idx_stat
|
||||
JOIN pg_index
|
||||
USING (indexrelid)
|
||||
JOIN pg_indexes as indexes
|
||||
ON idx_stat.schemaname = indexes.schemaname
|
||||
AND idx_stat.relname = indexes.tablename
|
||||
AND idx_stat.indexrelname = indexes.indexname
|
||||
WHERE pg_index.indisunique = FALSE
|
||||
),
|
||||
index_ratios AS (
|
||||
SELECT schemaname, tablename, indexname,
|
||||
idx_scan, all_scans,
|
||||
round(( CASE WHEN all_scans = 0 THEN 0.0::NUMERIC
|
||||
ELSE idx_scan::NUMERIC/all_scans * 100 END),2) as index_scan_pct,
|
||||
writes,
|
||||
round((CASE WHEN writes = 0 THEN idx_scan::NUMERIC ELSE idx_scan::NUMERIC/writes END),2)
|
||||
as scans_per_write,
|
||||
pg_size_pretty(index_bytes) as index_size,
|
||||
pg_size_pretty(table_size) as table_size,
|
||||
idx_is_btree, index_bytes
|
||||
FROM indexes
|
||||
JOIN table_scans
|
||||
USING (relid)
|
||||
),
|
||||
index_groups AS (
|
||||
SELECT 'Never Used Indexes' as reason, *, 1 as grp
|
||||
FROM index_ratios
|
||||
WHERE
|
||||
idx_scan = 0
|
||||
and idx_is_btree
|
||||
UNION ALL
|
||||
SELECT 'Low Scans, High Writes' as reason, *, 2 as grp
|
||||
FROM index_ratios
|
||||
WHERE
|
||||
scans_per_write <= 1
|
||||
and index_scan_pct < 10
|
||||
and idx_scan > 0
|
||||
and writes > 100
|
||||
and idx_is_btree
|
||||
UNION ALL
|
||||
SELECT 'Seldom Used Large Indexes' as reason, *, 3 as grp
|
||||
FROM index_ratios
|
||||
WHERE
|
||||
index_scan_pct < 5
|
||||
and scans_per_write > 1
|
||||
and idx_scan > 0
|
||||
and idx_is_btree
|
||||
and index_bytes > 100000000
|
||||
UNION ALL
|
||||
SELECT 'High-Write Large Non-Btree' as reason, index_ratios.*, 4 as grp
|
||||
FROM index_ratios, all_writes
|
||||
WHERE
|
||||
( writes::NUMERIC / ( total_writes + 1 ) ) > 0.02
|
||||
AND NOT idx_is_btree
|
||||
AND index_bytes > 100000000
|
||||
ORDER BY grp, index_bytes DESC )
|
||||
SELECT reason, schemaname, tablename, indexname,
|
||||
index_scan_pct, scans_per_write, index_size, table_size
|
||||
FROM index_groups;
|
||||
*/
|
||||
#endregion
|
||||
|
||||
static int startingSchema = -1;
|
||||
public static int currentSchema = -1;
|
||||
@@ -154,6 +237,8 @@ namespace AyaNova.Util
|
||||
"ayid bigint not null, ayatype integer not null, ayevent integer not null, textra varchar(255))");
|
||||
//INDEX: Most selective first as there is more unique ID's than unique types
|
||||
await ExecQueryAsync("CREATE INDEX aevent_typeid_idx ON aevent (ayid, ayatype);");
|
||||
|
||||
//TODO: this may be a very low used index, revisit it down the road
|
||||
await ExecQueryAsync("CREATE INDEX aevent_userid_idx ON aevent (userid);");
|
||||
|
||||
|
||||
@@ -216,10 +301,10 @@ $BODY$;
|
||||
|
||||
//create translation text tables
|
||||
await ExecQueryAsync("CREATE TABLE atranslation (id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, name varchar(255) not null, stock bool, cjkindex bool default false)");
|
||||
//LOOKAT: I don't think this is doing anything:
|
||||
//exec("CREATE UNIQUE INDEX atranslation_name_idx ON atranslation (name)");
|
||||
|
||||
await ExecQueryAsync("CREATE TABLE atranslationitem (id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, translationid bigint not null REFERENCES atranslation (id), key text not null, display text not null)");
|
||||
await ExecQueryAsync("CREATE INDEX atranslationitem_translationid_key_display_idx ON atranslationitem (translationid,key, display)");
|
||||
//This is not a well used index, not sure what it's point is
|
||||
// await ExecQueryAsync("CREATE INDEX atranslationitem_translationid_key_display_idx ON atranslationitem (translationid,key, display)");
|
||||
|
||||
//Load the default TRANSLATIONS
|
||||
await AyaNova.Biz.PrimeData.PrimeTranslations();
|
||||
|
||||
Reference in New Issue
Block a user