admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Browse Source
This commit is contained in:
John Cardinal
2020-05-20 22:04:51 +00:00
parent 1e2f072161
commit 64b61d14fa
2 changed files with 297 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
devdocs/todo.txt
View File

@@ -7,6 +7,54 @@ todo: Search indexing performance improvement and exception avoidance (Search.cs
Idea: do the insert manually with the clause "on conflict do nothing" Idea: do the insert manually with the clause "on conflict do nothing"
if detect it hasn't inserted (conflict) trigger a fetch instead if detect it hasn't inserted (conflict) trigger a fetch instead
like what is being done now but won't have the exception to deal with!! like what is being done now but won't have the exception to deal with!!
var CtAdd.SearchDictionary.FromSqlRaw("insert into asearchdictionary (word) values('{0}') on conflict (word) do update set word=excluded.word returning id",KeyWord ).FirstOrDefaultAsync();
stored procedure?
https://www.postgresqltutorial.com/plpgsql-loop-statements/
-------
-- PROCEDURE: public.pdoindex(text[], bigint, integer, boolean)
-- DROP PROCEDURE public.pdoindex(text[], bigint, integer, boolean);
CREATE OR REPLACE PROCEDURE public.pdoindex(
words text[],
objectid bigint,
objecttype integer,
isupdate boolean DEFAULT false)
LANGUAGE 'plpgsql'
AS $BODY$DECLARE
s text;
wordid bigint;
BEGIN
IF objectid=0 THEN
RAISE EXCEPTION 'Bad object id --> %', objectid;
END IF;
IF objecttype=0 THEN
RAISE EXCEPTION 'Bad object type --> %', objecttype;
END IF;
-- insert into asearchdictionary (word) values('{0}') on conflict (word) do update set word=excluded.word returning *
-- iterate text in loop
FOREACH s IN ARRAY words
LOOP
insert into asearchdictionary (word) values(s) on conflict (word) do update set word=excluded.word returning id into wordid;
insert into asearchkey (wordid,objectid,objecttype) values(wordid,objectid,objecttype);
--RAISE info 'word is %', s;
--RAISE info 'word id is %', wordid;
END LOOP;
END;
$BODY$;
-------
ALTER FUNCTION public.doindex(text[], bigint, integer, boolean)
OWNER TO postgres;
todo: Search confirm indexes are actually being used todo: Search confirm indexes are actually being used

403
server/AyaNova/biz/Search.cs
View File

@@ -747,105 +747,6 @@ WHERE a.word IN ('eos', 'quia', 'voluptate', 'delectus', 'sapiente', 'omnis', 's
} }
#region OLD - NEW WORD ADDITION second attempt, do it word by word and accept clashes and handle them
// //-------- START CRITICAL SECTION -----------
// //-------------------------------------------
// #if (DEBUG)
// var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###");
// #endif
// foreach (string KeyWord in KeyWordList)
// {
// if (!ExistingKeywordMatches.ContainsValue(KeyWord))
// {
// //algorithm: Attempt to add it to the db and get the id, if it fails with the expected exception for a duplicate word insertion attempt, then immediately read back that word and handle it
// //ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY
// SearchDictionary NewWord = new SearchDictionary();
// NewWord.Word = KeyWord;
// try
// {
// //ADD WORD TO DICTIONARY, SAVE THE ID INTO THE MATCHINGKEYWORDIDLIST
// var CtAdd = ServiceProviderProvider.DBContext;
// await CtAdd.SearchDictionary.AddAsync(NewWord);
// await CtAdd.SaveChangesAsync();
// //-------
// //Add to matching keywords
// MatchingKeywordIdList.Add(NewWord.Id);
// //-------
// //It exists now
// ExistingKeywordMatches.Add(NewWord.Id, NewWord.Word);
// }
// catch (Microsoft.EntityFrameworkCore.DbUpdateException ex)
// {
// #region Exceptions from word already existing (added maybe in another thread)
// #if (DEBUG)
// log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead...");
// #endif
// //FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED?
// if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx"))
// {
// #if (DEBUG)
// log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?");
// #endif
// throw ex;
// }
// //FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD
// var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync();
// if (SearchDictionaryMatchFoundInDB != null)
// {
// MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id);
// //It exists now
// ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word);
// }
// else
// {
// #if (DEBUG)
// log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?");
// #endif
// }
// #endregion
// }
// catch (Exception ex)
// {
// #if (DEBUG)
// log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?");
// #endif
// throw ex;
// }
// }
// }
// //-------- END CRITICAL SECTION -------------
// //-------------------------------------------
#endregion second attempt
#region NEW WORD ADDITION THIRD attempt, do it word by word use no conflict and bypass EF Core entirely
//-------- START CRITICAL SECTION -----------
//-------------------------------------------
#region PERFORMANCE NOTES / EXPERIMENTS #region PERFORMANCE NOTES / EXPERIMENTS
/* /*
@@ -975,84 +876,278 @@ RUN 6 - SMALL 2
*/ */
#endregion performance notes experiments #endregion performance notes experiments
#region OLD - NEW WORD ADDITION second attempt, do it word by word and accept clashes and handle them
// //-------- START CRITICAL SECTION -----------
// //-------------------------------------------
// #if (DEBUG)
// var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###");
// #endif
// foreach (string KeyWord in KeyWordList)
// {
// if (!ExistingKeywordMatches.ContainsValue(KeyWord))
// {
// //algorithm: Attempt to add it to the db and get the id, if it fails with the expected exception for a duplicate word insertion attempt, then immediately read back that word and handle it
// //ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY
// SearchDictionary NewWord = new SearchDictionary();
// NewWord.Word = KeyWord;
// try
// {
// //ADD WORD TO DICTIONARY, SAVE THE ID INTO THE MATCHINGKEYWORDIDLIST
// var CtAdd = ServiceProviderProvider.DBContext;
// await CtAdd.SearchDictionary.AddAsync(NewWord);
// await CtAdd.SaveChangesAsync();
// //-------
// //Add to matching keywords
// MatchingKeywordIdList.Add(NewWord.Id);
// //-------
// //It exists now
// ExistingKeywordMatches.Add(NewWord.Id, NewWord.Word);
// }
// catch (Microsoft.EntityFrameworkCore.DbUpdateException ex)
// {
// #region Exceptions from word already existing (added maybe in another thread)
// #if (DEBUG)
// log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead...");
// #endif
// //FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED?
// if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx"))
// {
// #if (DEBUG)
// log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?");
// #endif
// throw ex;
// }
// //FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD
// var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync();
// if (SearchDictionaryMatchFoundInDB != null)
// {
// MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id);
// //It exists now
// ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word);
// }
// else
// {
// #if (DEBUG)
// log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?");
// #endif
// }
// #endregion
// }
// catch (Exception ex)
// {
// #if (DEBUG)
// log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?");
// #endif
// throw ex;
// }
// }
// }
// //-------- END CRITICAL SECTION -------------
// //-------------------------------------------
#endregion second attempt
#region NEW WORD ADDITION THIRD attempt, do it word by word use no conflict and bypass EF Core entirely
// //-------- START CRITICAL SECTION -----------
// //-------------------------------------------
// #if (DEBUG)
// var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###");
// #endif
// var CtAdd = ServiceProviderProvider.DBContext;
// using (var command = CtAdd.Database.GetDbConnection().CreateCommand())
// {
// await CtAdd.Database.OpenConnectionAsync();
// foreach (string KeyWord in KeyWordList)
// {
// if (!ExistingKeywordMatches.ContainsValue(KeyWord))
// {
// //ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY
// try
// {
// //insert, if conflict then do what is essentially a no-op by updating word to it's existing value
// //always returning id, this is then in effect an upsert without an update since all we need is the id
// command.CommandText = $"insert into asearchdictionary (word) values('{KeyWord}') on conflict (word) do update set word=excluded.word returning id ";
// using (var dr = await command.ExecuteReaderAsync())
// {
// dr.Read();
// var wordId = dr.GetInt64(0);
// MatchingKeywordIdList.Add(wordId);
// ExistingKeywordMatches.Add(wordId, KeyWord);
// }
// }
// catch (Microsoft.EntityFrameworkCore.DbUpdateException ex)
// {
// #region Exceptions from word already existing (added maybe in another thread)
// #if (DEBUG)
// log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead...");
// #endif
// //FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED?
// if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx"))
// {
// #if (DEBUG)
// log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?");
// #endif
// throw ex;
// }
// //FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD
// var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync();
// if (SearchDictionaryMatchFoundInDB != null)
// {
// MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id);
// //It exists now
// ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word);
// }
// else
// {
// #if (DEBUG)
// log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?");
// #endif
// }
// #endregion
// }
// catch (Exception ex)
// {
// #if (DEBUG)
// log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?");
// #endif
// throw ex;
// }
// }
// }
// }//end of db using statement
// //-------- END CRITICAL SECTION -------------
// //-------------------------------------------
#endregion third attempt
#region FOURTH ATTEMPT NEW WORD ADDITION -hybrid using efcore but raw sql
//-------- START CRITICAL SECTION -----------
//-------------------------------------------
#if (DEBUG) #if (DEBUG)
var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###"); var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###");
#endif #endif
var CtAdd = ServiceProviderProvider.DBContext;
using (var command = CtAdd.Database.GetDbConnection().CreateCommand()) foreach (string KeyWord in KeyWordList)
{ {
if (!ExistingKeywordMatches.ContainsValue(KeyWord))
await CtAdd.Database.OpenConnectionAsync();
foreach (string KeyWord in KeyWordList)
{ {
if (!ExistingKeywordMatches.ContainsValue(KeyWord))
//algorithm: Attempt to add it to the db and get the id, if it fails with the expected exception for a duplicate word insertion attempt, then immediately read back that word and handle it
//ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY
SearchDictionary NewWord = new SearchDictionary();
NewWord.Word = KeyWord;
try
{ {
//ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY
try
{
//insert, if conflict then do what is essentially a no-op by updating word to it's existing value
//always returning id, this is then in effect an upsert without an update since all we need is the id
command.CommandText = $"insert into asearchdictionary (word) values('{KeyWord}') on conflict (word) do update set word=excluded.word returning id ";
using (var dr = await command.ExecuteReaderAsync())
{
dr.Read();
var wordId = dr.GetInt64(0);
MatchingKeywordIdList.Add(wordId);
ExistingKeywordMatches.Add(wordId, KeyWord);
}
}
catch (Microsoft.EntityFrameworkCore.DbUpdateException ex)
{
#region Exceptions from word already existing (added maybe in another thread)
#if (DEBUG)
log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead...");
#endif
//FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED?
if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx"))
{
#if (DEBUG)
log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?");
#endif
throw ex;
}
//FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD //ADD WORD TO DICTIONARY, SAVE THE ID INTO THE MATCHINGKEYWORDIDLIST
var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync(); var CtAdd = ServiceProviderProvider.DBContext;
if (SearchDictionaryMatchFoundInDB != null) // var v = await CtAdd.SearchDictionary.FromSqlRaw("insert into asearchdictionary (word) values('{0}') on conflict (word) do update set word=excluded.word returning *", KeyWord).FirstOrDefaultAsync();
{
MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id); var v = await CtAdd.SearchDictionary.FromSqlInterpolated($"insert into asearchdictionary (word) values('{KeyWord}') on conflict (word) do update set word=excluded.word returning *").AsNoTracking().ToListAsync();
//It exists now
ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word); //await CtAdd.SearchDictionary.AddAsync(NewWord);
} //await CtAdd.SaveChangesAsync();
else
{
//-------
//Add to matching keywords
MatchingKeywordIdList.Add(NewWord.Id);
//-------
//It exists now
ExistingKeywordMatches.Add(NewWord.Id, NewWord.Word);
}
catch (Microsoft.EntityFrameworkCore.DbUpdateException ex)
{
#region Exceptions from word already existing (added maybe in another thread)
#if (DEBUG) #if (DEBUG)
log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?"); log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead...");
#endif #endif
} //FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED?
#endregion if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx"))
}
catch (Exception ex)
{ {
#if (DEBUG) #if (DEBUG)
log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?"); log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?");
#endif #endif
throw ex; throw ex;
} }
//FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD
var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync();
if (SearchDictionaryMatchFoundInDB != null)
{
MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id);
//It exists now
ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word);
}
else
{
#if (DEBUG)
log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?");
#endif
}
#endregion
}
catch (Exception ex)
{
#if (DEBUG)
log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?");
#endif
throw ex;
} }
} }
}//end of db using statement }
//-------- END CRITICAL SECTION ------------- //-------- END CRITICAL SECTION -------------
//------------------------------------------- //-------------------------------------------
#endregion third attempt #endregion second attempt