admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Browse Source
This commit is contained in:
John Cardinal
2020-05-20 22:04:51 +00:00
parent 1e2f072161
commit 64b61d14fa
2 changed files with 297 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
devdocs/todo.txt
View File

@@ -7,6 +7,54 @@ todo: Search indexing performance improvement and exception avoidance (Search.cs
Idea: do the insert manually with the clause "on conflict do nothing"
if detect it hasn't inserted (conflict) trigger a fetch instead
like what is being done now but won't have the exception to deal with!!
var CtAdd.SearchDictionary.FromSqlRaw("insert into asearchdictionary (word) values('{0}') on conflict (word) do update set word=excluded.word returning id",KeyWord ).FirstOrDefaultAsync();
stored procedure?
https://www.postgresqltutorial.com/plpgsql-loop-statements/
-------
-- PROCEDURE: public.pdoindex(text[], bigint, integer, boolean)
-- DROP PROCEDURE public.pdoindex(text[], bigint, integer, boolean);
CREATE OR REPLACE PROCEDURE public.pdoindex(
words text[],
objectid bigint,
objecttype integer,
isupdate boolean DEFAULT false)
LANGUAGE 'plpgsql'
AS $BODY$DECLARE
s text;
wordid bigint;
BEGIN
IF objectid=0 THEN
RAISE EXCEPTION 'Bad object id --> %', objectid;
END IF;
IF objecttype=0 THEN
RAISE EXCEPTION 'Bad object type --> %', objecttype;
END IF;
-- insert into asearchdictionary (word) values('{0}') on conflict (word) do update set word=excluded.word returning *
-- iterate text in loop
FOREACH s IN ARRAY words
LOOP
insert into asearchdictionary (word) values(s) on conflict (word) do update set word=excluded.word returning id into wordid;
insert into asearchkey (wordid,objectid,objecttype) values(wordid,objectid,objecttype);
--RAISE info 'word is %', s;
--RAISE info 'word id is %', wordid;
END LOOP;
END;
$BODY$;
-------
ALTER FUNCTION public.doindex(text[], bigint, integer, boolean)
OWNER TO postgres;
todo: Search confirm indexes are actually being used

403
server/AyaNova/biz/Search.cs
View File

@@ -747,105 +747,6 @@ WHERE a.word IN ('eos', 'quia', 'voluptate', 'delectus', 'sapiente', 'omnis', 's
}
#region OLD - NEW WORD ADDITION second attempt, do it word by word and accept clashes and handle them
// //-------- START CRITICAL SECTION -----------
// //-------------------------------------------
// #if (DEBUG)
// var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###");
// #endif
// foreach (string KeyWord in KeyWordList)
// {
// if (!ExistingKeywordMatches.ContainsValue(KeyWord))
// {
// //algorithm: Attempt to add it to the db and get the id, if it fails with the expected exception for a duplicate word insertion attempt, then immediately read back that word and handle it
// //ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY
// SearchDictionary NewWord = new SearchDictionary();
// NewWord.Word = KeyWord;
// try
// {
// //ADD WORD TO DICTIONARY, SAVE THE ID INTO THE MATCHINGKEYWORDIDLIST
// var CtAdd = ServiceProviderProvider.DBContext;
// await CtAdd.SearchDictionary.AddAsync(NewWord);
// await CtAdd.SaveChangesAsync();
// //-------
// //Add to matching keywords
// MatchingKeywordIdList.Add(NewWord.Id);
// //-------
// //It exists now
// ExistingKeywordMatches.Add(NewWord.Id, NewWord.Word);
// }
// catch (Microsoft.EntityFrameworkCore.DbUpdateException ex)
// {
// #region Exceptions from word already existing (added maybe in another thread)
// #if (DEBUG)
// log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead...");
// #endif
// //FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED?
// if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx"))
// {
// #if (DEBUG)
// log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?");
// #endif
// throw ex;
// }
// //FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD
// var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync();
// if (SearchDictionaryMatchFoundInDB != null)
// {
// MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id);
// //It exists now
// ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word);
// }
// else
// {
// #if (DEBUG)
// log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?");
// #endif
// }
// #endregion
// }
// catch (Exception ex)
// {
// #if (DEBUG)
// log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?");
// #endif
// throw ex;
// }
// }
// }
// //-------- END CRITICAL SECTION -------------
// //-------------------------------------------
#endregion second attempt
#region NEW WORD ADDITION THIRD attempt, do it word by word use no conflict and bypass EF Core entirely
//-------- START CRITICAL SECTION -----------
//-------------------------------------------
#region PERFORMANCE NOTES / EXPERIMENTS
/*
@@ -975,84 +876,278 @@ RUN 6 - SMALL 2
*/
#endregion performance notes experiments
#region OLD - NEW WORD ADDITION second attempt, do it word by word and accept clashes and handle them
// //-------- START CRITICAL SECTION -----------
// //-------------------------------------------
// #if (DEBUG)
// var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###");
// #endif
// foreach (string KeyWord in KeyWordList)
// {
// if (!ExistingKeywordMatches.ContainsValue(KeyWord))
// {
// //algorithm: Attempt to add it to the db and get the id, if it fails with the expected exception for a duplicate word insertion attempt, then immediately read back that word and handle it
// //ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY
// SearchDictionary NewWord = new SearchDictionary();
// NewWord.Word = KeyWord;
// try
// {
// //ADD WORD TO DICTIONARY, SAVE THE ID INTO THE MATCHINGKEYWORDIDLIST
// var CtAdd = ServiceProviderProvider.DBContext;
// await CtAdd.SearchDictionary.AddAsync(NewWord);
// await CtAdd.SaveChangesAsync();
// //-------
// //Add to matching keywords
// MatchingKeywordIdList.Add(NewWord.Id);
// //-------
// //It exists now
// ExistingKeywordMatches.Add(NewWord.Id, NewWord.Word);
// }
// catch (Microsoft.EntityFrameworkCore.DbUpdateException ex)
// {
// #region Exceptions from word already existing (added maybe in another thread)
// #if (DEBUG)
// log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead...");
// #endif
// //FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED?
// if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx"))
// {
// #if (DEBUG)
// log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?");
// #endif
// throw ex;
// }
// //FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD
// var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync();
// if (SearchDictionaryMatchFoundInDB != null)
// {
// MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id);
// //It exists now
// ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word);
// }
// else
// {
// #if (DEBUG)
// log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?");
// #endif
// }
// #endregion
// }
// catch (Exception ex)
// {
// #if (DEBUG)
// log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?");
// #endif
// throw ex;
// }
// }
// }
// //-------- END CRITICAL SECTION -------------
// //-------------------------------------------
#endregion second attempt
#region NEW WORD ADDITION THIRD attempt, do it word by word use no conflict and bypass EF Core entirely
// //-------- START CRITICAL SECTION -----------
// //-------------------------------------------
// #if (DEBUG)
// var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###");
// #endif
// var CtAdd = ServiceProviderProvider.DBContext;
// using (var command = CtAdd.Database.GetDbConnection().CreateCommand())
// {
// await CtAdd.Database.OpenConnectionAsync();
// foreach (string KeyWord in KeyWordList)
// {
// if (!ExistingKeywordMatches.ContainsValue(KeyWord))
// {
// //ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY
// try
// {
// //insert, if conflict then do what is essentially a no-op by updating word to it's existing value
// //always returning id, this is then in effect an upsert without an update since all we need is the id
// command.CommandText = $"insert into asearchdictionary (word) values('{KeyWord}') on conflict (word) do update set word=excluded.word returning id ";
// using (var dr = await command.ExecuteReaderAsync())
// {
// dr.Read();
// var wordId = dr.GetInt64(0);
// MatchingKeywordIdList.Add(wordId);
// ExistingKeywordMatches.Add(wordId, KeyWord);
// }
// }
// catch (Microsoft.EntityFrameworkCore.DbUpdateException ex)
// {
// #region Exceptions from word already existing (added maybe in another thread)
// #if (DEBUG)
// log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead...");
// #endif
// //FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED?
// if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx"))
// {
// #if (DEBUG)
// log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?");
// #endif
// throw ex;
// }
// //FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD
// var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync();
// if (SearchDictionaryMatchFoundInDB != null)
// {
// MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id);
// //It exists now
// ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word);
// }
// else
// {
// #if (DEBUG)
// log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?");
// #endif
// }
// #endregion
// }
// catch (Exception ex)
// {
// #if (DEBUG)
// log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?");
// #endif
// throw ex;
// }
// }
// }
// }//end of db using statement
// //-------- END CRITICAL SECTION -------------
// //-------------------------------------------
#endregion third attempt
#region FOURTH ATTEMPT NEW WORD ADDITION -hybrid using efcore but raw sql
//-------- START CRITICAL SECTION -----------
//-------------------------------------------
#if (DEBUG)
var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###");
#endif
var CtAdd = ServiceProviderProvider.DBContext;
using (var command = CtAdd.Database.GetDbConnection().CreateCommand())
foreach (string KeyWord in KeyWordList)
{
await CtAdd.Database.OpenConnectionAsync();
foreach (string KeyWord in KeyWordList)
if (!ExistingKeywordMatches.ContainsValue(KeyWord))
{
if (!ExistingKeywordMatches.ContainsValue(KeyWord))
//algorithm: Attempt to add it to the db and get the id, if it fails with the expected exception for a duplicate word insertion attempt, then immediately read back that word and handle it
//ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY
SearchDictionary NewWord = new SearchDictionary();
NewWord.Word = KeyWord;
try
{
//ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY
try
{
//insert, if conflict then do what is essentially a no-op by updating word to it's existing value
//always returning id, this is then in effect an upsert without an update since all we need is the id
command.CommandText = $"insert into asearchdictionary (word) values('{KeyWord}') on conflict (word) do update set word=excluded.word returning id ";
using (var dr = await command.ExecuteReaderAsync())
{
dr.Read();
var wordId = dr.GetInt64(0);
MatchingKeywordIdList.Add(wordId);
ExistingKeywordMatches.Add(wordId, KeyWord);
}
}
catch (Microsoft.EntityFrameworkCore.DbUpdateException ex)
{
#region Exceptions from word already existing (added maybe in another thread)
#if (DEBUG)
log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead...");
#endif
//FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED?
if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx"))
{
#if (DEBUG)
log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?");
#endif
throw ex;
}
//FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD
var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync();
if (SearchDictionaryMatchFoundInDB != null)
{
MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id);
//It exists now
ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word);
}
else
{
//ADD WORD TO DICTIONARY, SAVE THE ID INTO THE MATCHINGKEYWORDIDLIST
var CtAdd = ServiceProviderProvider.DBContext;
// var v = await CtAdd.SearchDictionary.FromSqlRaw("insert into asearchdictionary (word) values('{0}') on conflict (word) do update set word=excluded.word returning *", KeyWord).FirstOrDefaultAsync();
var v = await CtAdd.SearchDictionary.FromSqlInterpolated($"insert into asearchdictionary (word) values('{KeyWord}') on conflict (word) do update set word=excluded.word returning *").AsNoTracking().ToListAsync();
//await CtAdd.SearchDictionary.AddAsync(NewWord);
//await CtAdd.SaveChangesAsync();
//-------
//Add to matching keywords
MatchingKeywordIdList.Add(NewWord.Id);
//-------
//It exists now
ExistingKeywordMatches.Add(NewWord.Id, NewWord.Word);
}
catch (Microsoft.EntityFrameworkCore.DbUpdateException ex)
{
#region Exceptions from word already existing (added maybe in another thread)
#if (DEBUG)
log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?");
log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead...");
#endif
}
#endregion
}
catch (Exception ex)
//FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED?
if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx"))
{
#if (DEBUG)
log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?");
log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?");
#endif
throw ex;
}
//FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD
var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync();
if (SearchDictionaryMatchFoundInDB != null)
{
MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id);
//It exists now
ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word);
}
else
{
#if (DEBUG)
log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?");
#endif
}
#endregion
}
catch (Exception ex)
{
#if (DEBUG)
log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?");
#endif
throw ex;
}
}
}//end of db using statement
}
//-------- END CRITICAL SECTION -------------
//-------------------------------------------
#endregion third attempt
#endregion second attempt