admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Browse Source
This commit is contained in:
John Cardinal
2020-05-21 00:05:09 +00:00
parent 52d7aca736
commit 3a91bf62d3
3 changed files with 126 additions and 260 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.vscode/launch.json vendored
View File

@@ -51,7 +51,7 @@
"AYANOVA_FOLDER_BACKUP_FILES": "c:\\temp\\RavenTestData\\backupfiles",
"AYANOVA_METRICS_USE_INFLUXDB": "false",
"AYANOVA_SERVER_TEST_MODE":"true",
"AYANOVA_SERVER_TEST_MODE_SEEDLEVEL":"small",
"AYANOVA_SERVER_TEST_MODE_SEEDLEVEL":"huge",
"AYANOVA_SERVER_TEST_MODE_TZ_OFFSET":"-7",
"AYANOVA_BACKUP_PG_DUMP_PATH":"C:\\data\\code\\PostgreSQLPortable_12.0\\App\\PgSQL\\bin\\"

348
server/AyaNova/biz/Search.cs
View File

@@ -709,12 +709,6 @@ namespace AyaNova.Biz
}
#endif
//marker for commit before teardown and redo
//IF NOT NEW, DELETE ALL EXISTING ENTRIES FOR OBJECT TYPE AND ID
if (!newRecord)
{
await ProcessDeletedObjectKeywordsAsync(p.ObjectId, p.ObjectType);
}
//BREAK OBJECT TEXT STRINGS INTO KEYWORD LIST
List<string> KeyWordList = await BreakAsync(p.TranslationId, p.Words);
@@ -727,10 +721,34 @@ namespace AyaNova.Biz
return;
}
//NOTES:
//Using slow but tried and true method:
//Highest ID in search dictionary is 830 count is 831
//Highest ID in search key is 18702 count is 18731
//Using v1.0 of new sp
//Highest id in search dictionary is 18728, count is 830 total
//Highest ID in search key is 18728 count is 18728
//Using v2.0 of new sp conflict do nothing then fetch
//Highest id in search dictionary is 18671, count is 829 total
//Highest ID in search key is 18671 count is 18671
//call stored procedure
var SomeValue = await ServiceProviderProvider.DBContext.Database.ExecuteSqlInterpolatedAsync($"call aydosearchindex({KeyWordList},{p.ObjectId},{p.ObjectType})");
return;
//BUILD A LIST OF MatchingDictionaryEntry items FOR THE MATCHING WORDS
List<long> MatchingKeywordIdList = new List<long>();
//******************* REDUNDANT PAST HERE *********************************
//ITERATE ALL THE KEYWORDS, SEARCH IN THE SEARCHDICTIONARY TABLE AND COLLECT ID'S OF ANY PRE-EXISTING IN DB KEYWORDS
var ExistingKeywordMatches = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => KeyWordList.Contains(z.Word)).ToDictionaryAsync(z => z.Id, z => z.Word);
/*example of above query, returns a list of words and ids
@@ -878,277 +896,95 @@ RUN 6 - SMALL 2
#region OLD - NEW WORD ADDITION second attempt, do it word by word and accept clashes and handle them
// //-------- START CRITICAL SECTION -----------
// //-------------------------------------------
//-------- START CRITICAL SECTION -----------
//-------------------------------------------
// #if (DEBUG)
// var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###");
// #endif
#if (DEBUG)
var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###");
#endif
// foreach (string KeyWord in KeyWordList)
// {
// if (!ExistingKeywordMatches.ContainsValue(KeyWord))
// {
foreach (string KeyWord in KeyWordList)
{
if (!ExistingKeywordMatches.ContainsValue(KeyWord))
{
// //algorithm: Attempt to add it to the db and get the id, if it fails with the expected exception for a duplicate word insertion attempt, then immediately read back that word and handle it
//algorithm: Attempt to add it to the db and get the id, if it fails with the expected exception for a duplicate word insertion attempt, then immediately read back that word and handle it
// //ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY
// SearchDictionary NewWord = new SearchDictionary();
// NewWord.Word = KeyWord;
//ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY
SearchDictionary NewWord = new SearchDictionary();
NewWord.Word = KeyWord;
// try
// {
try
{
// //ADD WORD TO DICTIONARY, SAVE THE ID INTO THE MATCHINGKEYWORDIDLIST
// var CtAdd = ServiceProviderProvider.DBContext;
// await CtAdd.SearchDictionary.AddAsync(NewWord);
// await CtAdd.SaveChangesAsync();
//ADD WORD TO DICTIONARY, SAVE THE ID INTO THE MATCHINGKEYWORDIDLIST
var CtAdd = ServiceProviderProvider.DBContext;
await CtAdd.SearchDictionary.AddAsync(NewWord);
await CtAdd.SaveChangesAsync();
// //-------
// //Add to matching keywords
// MatchingKeywordIdList.Add(NewWord.Id);
// //-------
//-------
//Add to matching keywords
MatchingKeywordIdList.Add(NewWord.Id);
//-------
// //It exists now
// ExistingKeywordMatches.Add(NewWord.Id, NewWord.Word);
// }
// catch (Microsoft.EntityFrameworkCore.DbUpdateException ex)
// {
// #region Exceptions from word already existing (added maybe in another thread)
// #if (DEBUG)
// log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead...");
// #endif
// //FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED?
// if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx"))
// {
// #if (DEBUG)
// log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?");
// #endif
// throw ex;
// }
//It exists now
ExistingKeywordMatches.Add(NewWord.Id, NewWord.Word);
}
catch (Microsoft.EntityFrameworkCore.DbUpdateException ex)
{
#region Exceptions from word already existing (added maybe in another thread)
#if (DEBUG)
log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead...");
#endif
//FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED?
if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx"))
{
#if (DEBUG)
log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?");
#endif
throw ex;
}
// //FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD
// var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync();
// if (SearchDictionaryMatchFoundInDB != null)
// {
// MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id);
// //It exists now
// ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word);
// }
// else
// {
// #if (DEBUG)
// log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?");
// #endif
// }
// #endregion
// }
// catch (Exception ex)
// {
// #if (DEBUG)
// log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?");
// #endif
// throw ex;
// }
// }
// }
//FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD
var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync();
if (SearchDictionaryMatchFoundInDB != null)
{
MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id);
//It exists now
ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word);
}
else
{
#if (DEBUG)
log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?");
#endif
}
#endregion
}
catch (Exception ex)
{
#if (DEBUG)
log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?");
#endif
throw ex;
}
}
}
// //-------- END CRITICAL SECTION -------------
// //-------------------------------------------
#endregion second attempt
#region NEW WORD ADDITION THIRD attempt, do it word by word use no conflict and bypass EF Core entirely
// //-------- START CRITICAL SECTION -----------
// //-------------------------------------------
// #if (DEBUG)
// var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###");
// #endif
// var CtAdd = ServiceProviderProvider.DBContext;
// using (var command = CtAdd.Database.GetDbConnection().CreateCommand())
// {
// await CtAdd.Database.OpenConnectionAsync();
// foreach (string KeyWord in KeyWordList)
// {
// if (!ExistingKeywordMatches.ContainsValue(KeyWord))
// {
// //ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY
// try
// {
// //insert, if conflict then do what is essentially a no-op by updating word to it's existing value
// //always returning id, this is then in effect an upsert without an update since all we need is the id
// command.CommandText = $"insert into asearchdictionary (word) values('{KeyWord}') on conflict (word) do update set word=excluded.word returning id ";
// using (var dr = await command.ExecuteReaderAsync())
// {
// dr.Read();
// var wordId = dr.GetInt64(0);
// MatchingKeywordIdList.Add(wordId);
// ExistingKeywordMatches.Add(wordId, KeyWord);
// }
// }
// catch (Microsoft.EntityFrameworkCore.DbUpdateException ex)
// {
// #region Exceptions from word already existing (added maybe in another thread)
// #if (DEBUG)
// log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead...");
// #endif
// //FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED?
// if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx"))
// {
// #if (DEBUG)
// log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?");
// #endif
// throw ex;
// }
// //FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD
// var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync();
// if (SearchDictionaryMatchFoundInDB != null)
// {
// MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id);
// //It exists now
// ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word);
// }
// else
// {
// #if (DEBUG)
// log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?");
// #endif
// }
// #endregion
// }
// catch (Exception ex)
// {
// #if (DEBUG)
// log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?");
// #endif
// throw ex;
// }
// }
// }
// }//end of db using statement
// //-------- END CRITICAL SECTION -------------
// //-------------------------------------------
#endregion third attempt
#region FOURTH ATTEMPT NEW WORD ADDITION -hybrid using efcore but raw sql
//-------- START CRITICAL SECTION -----------
//-------------------------------------------
#if (DEBUG)
var log = AyaNova.Util.ApplicationLogging.CreateLogger("### Search::ProcessKeywords ###");
#endif
foreach (string KeyWord in KeyWordList)
{
if (!ExistingKeywordMatches.ContainsValue(KeyWord))
{
//algorithm: Attempt to add it to the db and get the id, if it fails with the expected exception for a duplicate word insertion attempt, then immediately read back that word and handle it
//ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY
SearchDictionary NewWord = new SearchDictionary();
NewWord.Word = KeyWord;
try
{
//ADD WORD TO DICTIONARY, SAVE THE ID INTO THE MATCHINGKEYWORDIDLIST
var CtAdd = ServiceProviderProvider.DBContext;
// var v = await CtAdd.SearchDictionary.FromSqlRaw("insert into asearchdictionary (word) values('{0}') on conflict (word) do update set word=excluded.word returning *", KeyWord).FirstOrDefaultAsync();
var v = await CtAdd.SearchDictionary.FromSqlInterpolated($"insert into asearchdictionary (word) values('{KeyWord}') on conflict (word) do update set word=excluded.word returning *").AsNoTracking().ToListAsync();
//await CtAdd.SearchDictionary.AddAsync(NewWord);
//await CtAdd.SaveChangesAsync();
//-------
//Add to matching keywords
MatchingKeywordIdList.Add(NewWord.Id);
//-------
//It exists now
ExistingKeywordMatches.Add(NewWord.Id, NewWord.Word);
}
catch (Microsoft.EntityFrameworkCore.DbUpdateException ex)
{
#region Exceptions from word already existing (added maybe in another thread)
#if (DEBUG)
log.LogInformation($"###################### Exception caught attempting to add word: '{KeyWord}' fetching instead...");
#endif
//FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED?
if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx"))
{
#if (DEBUG)
log.LogInformation($"###################### Unexpected inner exception on add word: '{KeyWord}'!?");
#endif
throw ex;
}
//FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD
var SearchDictionaryMatchFoundInDB = await ServiceProviderProvider.DBContext.SearchDictionary.AsNoTracking().Where(z => z.Word == KeyWord).FirstOrDefaultAsync();
if (SearchDictionaryMatchFoundInDB != null)
{
MatchingKeywordIdList.Add(SearchDictionaryMatchFoundInDB.Id);
//It exists now
ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word);
}
else
{
#if (DEBUG)
log.LogInformation($"###################### NULL when expected to find word: '{KeyWord}'!?");
#endif
}
#endregion
}
catch (Exception ex)
{
#if (DEBUG)
log.LogInformation(ex, $"###################### Unexpected exception adding word: '{KeyWord}'!?");
#endif
throw ex;
}
}
}
//-------- END CRITICAL SECTION -------------
//-------------------------------------------
//-------- END CRITICAL SECTION -------------
//-------------------------------------------
#endregion second attempt

36
server/AyaNova/util/AySchema.cs
View File

@@ -155,7 +155,7 @@ namespace AyaNova.Util
//SEARCH TABLES
await ExecQueryAsync("CREATE TABLE asearchdictionary (id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, word varchar(255) not null)");
//Must be unique and also this is hit a *lot* during searches and also indexing
await ExecQueryAsync("CREATE UNIQUE INDEX asearchdictionary_word_idx ON asearchdictionary (word);");
@@ -173,7 +173,36 @@ namespace AyaNova.Util
await ExecQueryAsync("CREATE INDEX asearchkey_wordid_otype_idx ON asearchkey (wordid, objecttype);");
//Search indexing stored procedure
await ExecQueryAsync(@"CREATE OR REPLACE PROCEDURE public.aydosearchindex(
// await ExecQueryAsync(@"CREATE OR REPLACE PROCEDURE public.aydosearchindex(
// wordlist text[],
// ayobjectid bigint,
// ayobjecttype integer)
// LANGUAGE 'plpgsql'
// AS $BODY$DECLARE
// s text;
// wordid bigint;
// BEGIN
// IF ayobjectid=0 THEN
// RAISE EXCEPTION 'Bad object id --> %', ayobjectid;
// END IF;
// IF ayobjecttype=0 THEN
// RAISE EXCEPTION 'Bad object type --> %', ayobjecttype;
// END IF;
// delete from asearchkey where objectid=ayobjectid and objecttype=ayobjecttype;
// FOREACH s IN ARRAY wordlist
// LOOP
// insert into asearchdictionary (word) values(s) on conflict (word) do update set word=excluded.word returning id into wordid;
// insert into asearchkey (wordid,objectid,objecttype) values(wordid,ayobjectid,ayobjecttype);
// END LOOP;
// END;
// $BODY$;");
await ExecQueryAsync(@"CREATE OR REPLACE PROCEDURE public.aydosearchindex(
wordlist text[],
ayobjectid bigint,
ayobjecttype integer)
@@ -196,7 +225,8 @@ delete from asearchkey where objectid=ayobjectid and objecttype=ayobjecttype;
FOREACH s IN ARRAY wordlist
LOOP
insert into asearchdictionary (word) values(s) on conflict (word) do update set word=excluded.word returning id into wordid;
insert into asearchdictionary (word) values(s) on conflict (word) do nothing;
SELECT id INTO STRICT wordid FROM asearchdictionary WHERE word = s;
insert into asearchkey (wordid,objectid,objecttype) values(wordid,ayobjectid,ayobjecttype);
END LOOP;
END;