From 3a91bf62d34cd90e0e52c91a317c3196d030cb8b Mon Sep 17 00:00:00 2001 From: John Cardinal Date: Thu, 21 May 2020 00:05:09 +0000 Subject: [PATCH] --- .vscode/launch.json | 2 +- server/AyaNova/biz/Search.cs | 348 +++++++++----------------------- server/AyaNova/util/AySchema.cs | 36 +++- 3 files changed, 126 insertions(+), 260 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 6b9a0d68..259171c9 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -51,7 +51,7 @@ "AYANOVA_FOLDER_BACKUP_FILES": "c:\\temp\\RavenTestData\\backupfiles", "AYANOVA_METRICS_USE_INFLUXDB": "false", "AYANOVA_SERVER_TEST_MODE":"true", - "AYANOVA_SERVER_TEST_MODE_SEEDLEVEL":"small", + "AYANOVA_SERVER_TEST_MODE_SEEDLEVEL":"huge", "AYANOVA_SERVER_TEST_MODE_TZ_OFFSET":"-7", "AYANOVA_BACKUP_PG_DUMP_PATH":"C:\\data\\code\\PostgreSQLPortable_12.0\\App\\PgSQL\\bin\\" diff --git a/server/AyaNova/biz/Search.cs b/server/AyaNova/biz/Search.cs index 90cdfb4b..f0f72803 100644 --- a/server/AyaNova/biz/Search.cs +++ b/server/AyaNova/biz/Search.cs @@ -709,12 +709,6 @@ namespace AyaNova.Biz } #endif -//marker for commit before teardown and redo - //IF NOT NEW, DELETE ALL EXISTING ENTRIES FOR OBJECT TYPE AND ID - if (!newRecord) - { - await ProcessDeletedObjectKeywordsAsync(p.ObjectId, p.ObjectType); - } //BREAK OBJECT TEXT STRINGS INTO KEYWORD LIST List KeyWordList = await BreakAsync(p.TranslationId, p.Words); @@ -727,10 +721,34 @@ namespace AyaNova.Biz return; } +//NOTES: +//Using slow but tried and true method: +//Highest ID in search dictionary is 830 count is 831 +//Highest ID in search key is 18702 count is 18731 + +//Using v1.0 of new sp +//Highest id in search dictionary is 18728, count is 830 total +//Highest ID in search key is 18728 count is 18728 + + +//Using v2.0 of new sp conflict do nothing then fetch +//Highest id in search dictionary is 18671, count is 829 total +//Highest ID in search key is 18671 count is 18671 + + + //call stored procedure + var SomeValue = await ServiceProviderProvider.DBContext.Database.ExecuteSqlInterpolatedAsync($"call aydosearchindex({KeyWordList},{p.ObjectId},{p.ObjectType})"); + return; + //BUILD A LIST OF MatchingDictionaryEntry items FOR THE MATCHING WORDS List MatchingKeywordIdList = new List(); + + + + //******************* REDUNDANT PAST HERE ********************************* + //ITERATE ALL THE KEYWORDS, SEARCH IN THE SEARCHDICTIONARY TABLE AND COLLECT ID'S OF ANY PRE-EXISTING IN DB KEYWORDS var ExistingKeywordMatches = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => KeyWordList.Contains(z.Word)).ToDictionaryAsync(z => z.Id, z => z.Word); /*example of above query, returns a list of words and ids @@ -878,277 +896,95 @@ RUN 6 - SMALL 2 #region OLD - NEW WORD ADDITION second attempt, do it word by word and accept clashes and handle them - // //-------- START CRITICAL SECTION ----------- - // //------------------------------------------- + //-------- START CRITICAL SECTION ----------- + //------------------------------------------- - // #if (DEBUG) - // var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###"); - // #endif + #if (DEBUG) + var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###"); + #endif - // foreach (string KeyWord in KeyWordList) - // { - // if (!ExistingKeywordMatches.ContainsValue(KeyWord)) - // { + foreach (string KeyWord in KeyWordList) + { + if (!ExistingKeywordMatches.ContainsValue(KeyWord)) + { - // //algorithm: Attempt to add it to the db and get the id, if it fails with the expected exception for a duplicate word insertion attempt, then immediately read back that word and handle it + //algorithm: Attempt to add it to the db and get the id, if it fails with the expected exception for a duplicate word insertion attempt, then immediately read back that word and handle it - // //ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY - // SearchDictionary NewWord = new SearchDictionary(); - // NewWord.Word = KeyWord; + //ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY + SearchDictionary NewWord = new SearchDictionary(); + NewWord.Word = KeyWord; - // try - // { + try + { - // //ADD WORD TO DICTIONARY, SAVE THE ID INTO THE MATCHINGKEYWORDIDLIST - // var CtAdd = ServiceProviderProvider.DBContext; - // await CtAdd.SearchDictionary.AddAsync(NewWord); - // await CtAdd.SaveChangesAsync(); + //ADD WORD TO DICTIONARY, SAVE THE ID INTO THE MATCHINGKEYWORDIDLIST + var CtAdd = ServiceProviderProvider.DBContext; + await CtAdd.SearchDictionary.AddAsync(NewWord); + await CtAdd.SaveChangesAsync(); - // //------- - // //Add to matching keywords - // MatchingKeywordIdList.Add(NewWord.Id); - // //------- + //------- + //Add to matching keywords + MatchingKeywordIdList.Add(NewWord.Id); + //------- - // //It exists now - // ExistingKeywordMatches.Add(NewWord.Id, NewWord.Word); - // } - // catch (Microsoft.EntityFrameworkCore.DbUpdateException ex) - // { - // #region Exceptions from word already existing (added maybe in another thread) - // #if (DEBUG) - // log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead..."); - // #endif - // //FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED? - // if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx")) - // { - // #if (DEBUG) - // log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?"); - // #endif - // throw ex; - // } + //It exists now + ExistingKeywordMatches.Add(NewWord.Id, NewWord.Word); + } + catch (Microsoft.EntityFrameworkCore.DbUpdateException ex) + { + #region Exceptions from word already existing (added maybe in another thread) + #if (DEBUG) + log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead..."); + #endif + //FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED? + if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx")) + { + #if (DEBUG) + log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?"); + #endif + throw ex; + } - // //FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD - // var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync(); - // if (SearchDictionaryMatchFoundInDB != null) - // { - // MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id); - // //It exists now - // ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word); - // } - // else - // { - // #if (DEBUG) - // log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?"); - // #endif - // } - // #endregion - // } - // catch (Exception ex) - // { - // #if (DEBUG) - // log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?"); - // #endif - // throw ex; - // } - // } - // } + //FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD + var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync(); + if (SearchDictionaryMatchFoundInDB != null) + { + MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id); + //It exists now + ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word); + } + else + { + #if (DEBUG) + log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?"); + #endif + } + #endregion + } + catch (Exception ex) + { + #if (DEBUG) + log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?"); + #endif + throw ex; + } + } + } - // //-------- END CRITICAL SECTION ------------- - // //------------------------------------------- - #endregion second attempt - - #region NEW WORD ADDITION THIRD attempt, do it word by word use no conflict and bypass EF Core entirely - - // //-------- START CRITICAL SECTION ----------- - // //------------------------------------------- - - // #if (DEBUG) - // var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###"); - // #endif - - // var CtAdd = ServiceProviderProvider.DBContext; - // using (var command = CtAdd.Database.GetDbConnection().CreateCommand()) - // { - - // await CtAdd.Database.OpenConnectionAsync(); - - - // foreach (string KeyWord in KeyWordList) - // { - // if (!ExistingKeywordMatches.ContainsValue(KeyWord)) - // { - // //ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY - // try - // { - - // //insert, if conflict then do what is essentially a no-op by updating word to it's existing value - // //always returning id, this is then in effect an upsert without an update since all we need is the id - // command.CommandText = $"insert into asearchdictionary (word) values('{KeyWord}') on conflict (word) do update set word=excluded.word returning id "; - - // using (var dr = await command.ExecuteReaderAsync()) - // { - // dr.Read(); - // var wordId = dr.GetInt64(0); - // MatchingKeywordIdList.Add(wordId); - // ExistingKeywordMatches.Add(wordId, KeyWord); - // } - - - - // } - // catch (Microsoft.EntityFrameworkCore.DbUpdateException ex) - // { - // #region Exceptions from word already existing (added maybe in another thread) - // #if (DEBUG) - // log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead..."); - // #endif - // //FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED? - // if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx")) - // { - // #if (DEBUG) - // log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?"); - // #endif - // throw ex; - // } - - // //FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD - // var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync(); - // if (SearchDictionaryMatchFoundInDB != null) - // { - // MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id); - // //It exists now - // ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word); - // } - // else - // { - // #if (DEBUG) - // log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?"); - // #endif - // } - // #endregion - // } - // catch (Exception ex) - // { - // #if (DEBUG) - // log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?"); - // #endif - // throw ex; - // } - // } - // } - // }//end of db using statement - - - - // //-------- END CRITICAL SECTION ------------- - // //------------------------------------------- - #endregion third attempt - - - - #region FOURTH ATTEMPT NEW WORD ADDITION -hybrid using efcore but raw sql - - //-------- START CRITICAL SECTION ----------- - //------------------------------------------- - -#if (DEBUG) - var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###"); -#endif - - - foreach (string KeyWord in KeyWordList) - { - if (!ExistingKeywordMatches.ContainsValue(KeyWord)) - { - - - - - - //algorithm: Attempt to add it to the db and get the id, if it fails with the expected exception for a duplicate word insertion attempt, then immediately read back that word and handle it - - //ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY - SearchDictionary NewWord = new SearchDictionary(); - NewWord.Word = KeyWord; - - try - { - - - //ADD WORD TO DICTIONARY, SAVE THE ID INTO THE MATCHINGKEYWORDIDLIST - var CtAdd = ServiceProviderProvider.DBContext; - // var v = await CtAdd.SearchDictionary.FromSqlRaw("insert into asearchdictionary (word) values('{0}') on conflict (word) do update set word=excluded.word returning *", KeyWord).FirstOrDefaultAsync(); - - var v = await CtAdd.SearchDictionary.FromSqlInterpolated($"insert into asearchdictionary (word) values('{KeyWord}') on conflict (word) do update set word=excluded.word returning *").AsNoTracking().ToListAsync(); - - //await CtAdd.SearchDictionary.AddAsync(NewWord); - //await CtAdd.SaveChangesAsync(); - - - //------- - //Add to matching keywords - MatchingKeywordIdList.Add(NewWord.Id); - //------- - - //It exists now - ExistingKeywordMatches.Add(NewWord.Id, NewWord.Word); - } - catch (Microsoft.EntityFrameworkCore.DbUpdateException ex) - { - #region Exceptions from word already existing (added maybe in another thread) -#if (DEBUG) - log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead..."); -#endif - //FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED? - if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx")) - { -#if (DEBUG) - log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?"); -#endif - throw ex; - } - - //FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD - var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync(); - if (SearchDictionaryMatchFoundInDB != null) - { - MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id); - //It exists now - ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word); - } - else - { -#if (DEBUG) - log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?"); -#endif - } - #endregion - } - catch (Exception ex) - { -#if (DEBUG) - log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?"); -#endif - throw ex; - } - } - } - - - - //-------- END CRITICAL SECTION ------------- - //------------------------------------------- + //-------- END CRITICAL SECTION ------------- + //------------------------------------------- #endregion second attempt + diff --git a/server/AyaNova/util/AySchema.cs b/server/AyaNova/util/AySchema.cs index bca48b5b..294530f0 100644 --- a/server/AyaNova/util/AySchema.cs +++ b/server/AyaNova/util/AySchema.cs @@ -155,7 +155,7 @@ namespace AyaNova.Util //SEARCH TABLES - + await ExecQueryAsync("CREATE TABLE asearchdictionary (id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, word varchar(255) not null)"); //Must be unique and also this is hit a *lot* during searches and also indexing await ExecQueryAsync("CREATE UNIQUE INDEX asearchdictionary_word_idx ON asearchdictionary (word);"); @@ -173,7 +173,36 @@ namespace AyaNova.Util await ExecQueryAsync("CREATE INDEX asearchkey_wordid_otype_idx ON asearchkey (wordid, objecttype);"); //Search indexing stored procedure - await ExecQueryAsync(@"CREATE OR REPLACE PROCEDURE public.aydosearchindex( +// await ExecQueryAsync(@"CREATE OR REPLACE PROCEDURE public.aydosearchindex( +// wordlist text[], +// ayobjectid bigint, +// ayobjecttype integer) +// LANGUAGE 'plpgsql' + +// AS $BODY$DECLARE +// s text; +// wordid bigint; +// BEGIN +// IF ayobjectid=0 THEN +// RAISE EXCEPTION 'Bad object id --> %', ayobjectid; +// END IF; + +// IF ayobjecttype=0 THEN +// RAISE EXCEPTION 'Bad object type --> %', ayobjecttype; +// END IF; + + +// delete from asearchkey where objectid=ayobjectid and objecttype=ayobjecttype; + +// FOREACH s IN ARRAY wordlist +// LOOP +// insert into asearchdictionary (word) values(s) on conflict (word) do update set word=excluded.word returning id into wordid; +// insert into asearchkey (wordid,objectid,objecttype) values(wordid,ayobjectid,ayobjecttype); +// END LOOP; +// END; +// $BODY$;"); + + await ExecQueryAsync(@"CREATE OR REPLACE PROCEDURE public.aydosearchindex( wordlist text[], ayobjectid bigint, ayobjecttype integer) @@ -196,7 +225,8 @@ delete from asearchkey where objectid=ayobjectid and objecttype=ayobjecttype; FOREACH s IN ARRAY wordlist LOOP - insert into asearchdictionary (word) values(s) on conflict (word) do update set word=excluded.word returning id into wordid; + insert into asearchdictionary (word) values(s) on conflict (word) do nothing; + SELECT id INTO STRICT wordid FROM asearchdictionary WHERE word = s; insert into asearchkey (wordid,objectid,objecttype) values(wordid,ayobjectid,ayobjecttype); END LOOP; END;