From 64b61d14fa1c1db3180211a2564a257eff9c88ab Mon Sep 17 00:00:00 2001 From: John Cardinal Date: Wed, 20 May 2020 22:04:51 +0000 Subject: [PATCH] --- devdocs/todo.txt | 48 +++++ server/AyaNova/biz/Search.cs | 403 ++++++++++++++++++++++------------- 2 files changed, 297 insertions(+), 154 deletions(-) diff --git a/devdocs/todo.txt b/devdocs/todo.txt index c77886f8..19cbd493 100644 --- a/devdocs/todo.txt +++ b/devdocs/todo.txt @@ -7,6 +7,54 @@ todo: Search indexing performance improvement and exception avoidance (Search.cs Idea: do the insert manually with the clause "on conflict do nothing" if detect it hasn't inserted (conflict) trigger a fetch instead like what is being done now but won't have the exception to deal with!! + var CtAdd.SearchDictionary.FromSqlRaw("insert into asearchdictionary (word) values('{0}') on conflict (word) do update set word=excluded.word returning id",KeyWord ).FirstOrDefaultAsync(); + +stored procedure? +https://www.postgresqltutorial.com/plpgsql-loop-statements/ +------- +-- PROCEDURE: public.pdoindex(text[], bigint, integer, boolean) + +-- DROP PROCEDURE public.pdoindex(text[], bigint, integer, boolean); + +CREATE OR REPLACE PROCEDURE public.pdoindex( + words text[], + objectid bigint, + objecttype integer, + isupdate boolean DEFAULT false) +LANGUAGE 'plpgsql' + +AS $BODY$DECLARE + s text; + wordid bigint; +BEGIN + IF objectid=0 THEN + RAISE EXCEPTION 'Bad object id --> %', objectid; + END IF; + + IF objecttype=0 THEN + RAISE EXCEPTION 'Bad object type --> %', objecttype; + END IF; + + + +-- insert into asearchdictionary (word) values('{0}') on conflict (word) do update set word=excluded.word returning * +-- iterate text in loop +FOREACH s IN ARRAY words + LOOP + insert into asearchdictionary (word) values(s) on conflict (word) do update set word=excluded.word returning id into wordid; + insert into asearchkey (wordid,objectid,objecttype) values(wordid,objectid,objecttype); + --RAISE info 'word is %', s; + --RAISE info 'word id is %', wordid; + END LOOP; +END; +$BODY$; + + +------- + +ALTER FUNCTION public.doindex(text[], bigint, integer, boolean) + OWNER TO postgres; + todo: Search confirm indexes are actually being used diff --git a/server/AyaNova/biz/Search.cs b/server/AyaNova/biz/Search.cs index 9b6f91a3..143ba774 100644 --- a/server/AyaNova/biz/Search.cs +++ b/server/AyaNova/biz/Search.cs @@ -747,105 +747,6 @@ WHERE a.word IN ('eos', 'quia', 'voluptate', 'delectus', 'sapiente', 'omnis', 's } - - #region OLD - NEW WORD ADDITION second attempt, do it word by word and accept clashes and handle them - - // //-------- START CRITICAL SECTION ----------- - // //------------------------------------------- - - // #if (DEBUG) - // var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###"); - // #endif - - - // foreach (string KeyWord in KeyWordList) - // { - // if (!ExistingKeywordMatches.ContainsValue(KeyWord)) - // { - - - - - - // //algorithm: Attempt to add it to the db and get the id, if it fails with the expected exception for a duplicate word insertion attempt, then immediately read back that word and handle it - - // //ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY - // SearchDictionary NewWord = new SearchDictionary(); - // NewWord.Word = KeyWord; - - // try - // { - - - // //ADD WORD TO DICTIONARY, SAVE THE ID INTO THE MATCHINGKEYWORDIDLIST - // var CtAdd = ServiceProviderProvider.DBContext; - // await CtAdd.SearchDictionary.AddAsync(NewWord); - // await CtAdd.SaveChangesAsync(); - - - // //------- - // //Add to matching keywords - // MatchingKeywordIdList.Add(NewWord.Id); - // //------- - - // //It exists now - // ExistingKeywordMatches.Add(NewWord.Id, NewWord.Word); - // } - // catch (Microsoft.EntityFrameworkCore.DbUpdateException ex) - // { - // #region Exceptions from word already existing (added maybe in another thread) - // #if (DEBUG) - // log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead..."); - // #endif - // //FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED? - // if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx")) - // { - // #if (DEBUG) - // log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?"); - // #endif - // throw ex; - // } - - // //FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD - // var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync(); - // if (SearchDictionaryMatchFoundInDB != null) - // { - // MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id); - // //It exists now - // ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word); - // } - // else - // { - // #if (DEBUG) - // log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?"); - // #endif - // } - // #endregion - // } - // catch (Exception ex) - // { - // #if (DEBUG) - // log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?"); - // #endif - // throw ex; - // } - // } - // } - - - - // //-------- END CRITICAL SECTION ------------- - // //------------------------------------------- - #endregion second attempt - - - - - - #region NEW WORD ADDITION THIRD attempt, do it word by word use no conflict and bypass EF Core entirely - - //-------- START CRITICAL SECTION ----------- - //------------------------------------------- #region PERFORMANCE NOTES / EXPERIMENTS /* @@ -975,84 +876,278 @@ RUN 6 - SMALL 2 */ #endregion performance notes experiments + #region OLD - NEW WORD ADDITION second attempt, do it word by word and accept clashes and handle them + + // //-------- START CRITICAL SECTION ----------- + // //------------------------------------------- + + // #if (DEBUG) + // var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###"); + // #endif + + + // foreach (string KeyWord in KeyWordList) + // { + // if (!ExistingKeywordMatches.ContainsValue(KeyWord)) + // { + + + + + + // //algorithm: Attempt to add it to the db and get the id, if it fails with the expected exception for a duplicate word insertion attempt, then immediately read back that word and handle it + + // //ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY + // SearchDictionary NewWord = new SearchDictionary(); + // NewWord.Word = KeyWord; + + // try + // { + + + // //ADD WORD TO DICTIONARY, SAVE THE ID INTO THE MATCHINGKEYWORDIDLIST + // var CtAdd = ServiceProviderProvider.DBContext; + // await CtAdd.SearchDictionary.AddAsync(NewWord); + // await CtAdd.SaveChangesAsync(); + + + // //------- + // //Add to matching keywords + // MatchingKeywordIdList.Add(NewWord.Id); + // //------- + + // //It exists now + // ExistingKeywordMatches.Add(NewWord.Id, NewWord.Word); + // } + // catch (Microsoft.EntityFrameworkCore.DbUpdateException ex) + // { + // #region Exceptions from word already existing (added maybe in another thread) + // #if (DEBUG) + // log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead..."); + // #endif + // //FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED? + // if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx")) + // { + // #if (DEBUG) + // log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?"); + // #endif + // throw ex; + // } + + // //FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD + // var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync(); + // if (SearchDictionaryMatchFoundInDB != null) + // { + // MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id); + // //It exists now + // ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word); + // } + // else + // { + // #if (DEBUG) + // log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?"); + // #endif + // } + // #endregion + // } + // catch (Exception ex) + // { + // #if (DEBUG) + // log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?"); + // #endif + // throw ex; + // } + // } + // } + + + + // //-------- END CRITICAL SECTION ------------- + // //------------------------------------------- + #endregion second attempt + + #region NEW WORD ADDITION THIRD attempt, do it word by word use no conflict and bypass EF Core entirely + + // //-------- START CRITICAL SECTION ----------- + // //------------------------------------------- + + // #if (DEBUG) + // var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###"); + // #endif + + // var CtAdd = ServiceProviderProvider.DBContext; + // using (var command = CtAdd.Database.GetDbConnection().CreateCommand()) + // { + + // await CtAdd.Database.OpenConnectionAsync(); + + + // foreach (string KeyWord in KeyWordList) + // { + // if (!ExistingKeywordMatches.ContainsValue(KeyWord)) + // { + // //ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY + // try + // { + + // //insert, if conflict then do what is essentially a no-op by updating word to it's existing value + // //always returning id, this is then in effect an upsert without an update since all we need is the id + // command.CommandText = $"insert into asearchdictionary (word) values('{KeyWord}') on conflict (word) do update set word=excluded.word returning id "; + + // using (var dr = await command.ExecuteReaderAsync()) + // { + // dr.Read(); + // var wordId = dr.GetInt64(0); + // MatchingKeywordIdList.Add(wordId); + // ExistingKeywordMatches.Add(wordId, KeyWord); + // } + + + + // } + // catch (Microsoft.EntityFrameworkCore.DbUpdateException ex) + // { + // #region Exceptions from word already existing (added maybe in another thread) + // #if (DEBUG) + // log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead..."); + // #endif + // //FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED? + // if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx")) + // { + // #if (DEBUG) + // log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?"); + // #endif + // throw ex; + // } + + // //FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD + // var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync(); + // if (SearchDictionaryMatchFoundInDB != null) + // { + // MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id); + // //It exists now + // ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word); + // } + // else + // { + // #if (DEBUG) + // log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?"); + // #endif + // } + // #endregion + // } + // catch (Exception ex) + // { + // #if (DEBUG) + // log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?"); + // #endif + // throw ex; + // } + // } + // } + // }//end of db using statement + + + + // //-------- END CRITICAL SECTION ------------- + // //------------------------------------------- + #endregion third attempt + + + + #region FOURTH ATTEMPT NEW WORD ADDITION -hybrid using efcore but raw sql + + //-------- START CRITICAL SECTION ----------- + //------------------------------------------- + #if (DEBUG) var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###"); #endif - var CtAdd = ServiceProviderProvider.DBContext; - using (var command = CtAdd.Database.GetDbConnection().CreateCommand()) + + foreach (string KeyWord in KeyWordList) { - - await CtAdd.Database.OpenConnectionAsync(); - - - foreach (string KeyWord in KeyWordList) + if (!ExistingKeywordMatches.ContainsValue(KeyWord)) { - if (!ExistingKeywordMatches.ContainsValue(KeyWord)) + + + + + + //algorithm: Attempt to add it to the db and get the id, if it fails with the expected exception for a duplicate word insertion attempt, then immediately read back that word and handle it + + //ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY + SearchDictionary NewWord = new SearchDictionary(); + NewWord.Word = KeyWord; + + try { - //ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY - try - { - //insert, if conflict then do what is essentially a no-op by updating word to it's existing value - //always returning id, this is then in effect an upsert without an update since all we need is the id - command.CommandText = $"insert into asearchdictionary (word) values('{KeyWord}') on conflict (word) do update set word=excluded.word returning id "; - using (var dr = await command.ExecuteReaderAsync()) - { - dr.Read(); - var wordId = dr.GetInt64(0); - MatchingKeywordIdList.Add(wordId); - ExistingKeywordMatches.Add(wordId, KeyWord); - } - } - catch (Microsoft.EntityFrameworkCore.DbUpdateException ex) - { - #region Exceptions from word already existing (added maybe in another thread) -#if (DEBUG) - log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead..."); -#endif - //FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED? - if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx")) - { -#if (DEBUG) - log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?"); -#endif - throw ex; - } - //FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD - var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync(); - if (SearchDictionaryMatchFoundInDB != null) - { - MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id); - //It exists now - ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word); - } - else - { + //ADD WORD TO DICTIONARY, SAVE THE ID INTO THE MATCHINGKEYWORDIDLIST + var CtAdd = ServiceProviderProvider.DBContext; + // var v = await CtAdd.SearchDictionary.FromSqlRaw("insert into asearchdictionary (word) values('{0}') on conflict (word) do update set word=excluded.word returning *", KeyWord).FirstOrDefaultAsync(); + + var v = await CtAdd.SearchDictionary.FromSqlInterpolated($"insert into asearchdictionary (word) values('{KeyWord}') on conflict (word) do update set word=excluded.word returning *").AsNoTracking().ToListAsync(); + + //await CtAdd.SearchDictionary.AddAsync(NewWord); + //await CtAdd.SaveChangesAsync(); + + + //------- + //Add to matching keywords + MatchingKeywordIdList.Add(NewWord.Id); + //------- + + //It exists now + ExistingKeywordMatches.Add(NewWord.Id, NewWord.Word); + } + catch (Microsoft.EntityFrameworkCore.DbUpdateException ex) + { + #region Exceptions from word already existing (added maybe in another thread) #if (DEBUG) - log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?"); + log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead..."); #endif - } - #endregion - } - catch (Exception ex) + //FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED? + if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx")) { #if (DEBUG) - log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?"); + log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?"); #endif throw ex; } + + //FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD + var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync(); + if (SearchDictionaryMatchFoundInDB != null) + { + MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id); + //It exists now + ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word); + } + else + { +#if (DEBUG) + log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?"); +#endif + } + #endregion + } + catch (Exception ex) + { +#if (DEBUG) + log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?"); +#endif + throw ex; } } - }//end of db using statement + } //-------- END CRITICAL SECTION ------------- //------------------------------------------- - #endregion third attempt - + #endregion second attempt