admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Browse Source
This commit is contained in:
John Cardinal
2018-10-10 18:40:32 +00:00
parent 88d87019c2
commit 6de6d43d0b
3 changed files with 113 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

158
server/AyaNova/biz/Search.cs
View File

@@ -425,13 +425,12 @@ namespace AyaNova.Biz
return;
}
//BUILD A LIST OF MatchingDictionaryEntry items FOR THE MATCHING WORDS
List<MatchingDictionaryEntry> MatchingKeywordIdList = new List<MatchingDictionaryEntry>();
//ITERATE ALL THE KEYWORDS, SEARCH IN THE SEARCHDICTIONARY TABLE AND COLLECT ID'S OF ANY PRE-EXISTING IN DB KEYWORDS
var ExistingKeywordMatches = ct.SearchDictionary.Where(m => KeyWordList.Contains(m.Word)).ToDictionary(m => m.Id, m => m.Word);
var ExistingKeywordMatches = ct.SearchDictionary.AsNoTracking().Where(m => KeyWordList.Contains(m.Word)).ToDictionary(m => m.Id, m => m.Word);
//Put the matching keyword ID's into the list
foreach (KeyValuePair<long, string> K in ExistingKeywordMatches)
{
@@ -441,69 +440,128 @@ namespace AyaNova.Biz
MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = K.Key, InName = IsName });
}
//ITERATE THROUGH THE KEYWORDS THAT DO *NOT* HAVE MATCHES IN THE SEARCHDICTIONARY AND ADD THEM TO THE SEARCH DICTIONARY, COLLECTING THEIR ID'S
bool NewWordsAdded = false;
var NewSearchDictionaryWordsList = new List<SearchDictionary>();
//-------- START CRITICAL SECTION -----------
//-------------------------------------------
#region NEW WORD ADDITION second attempt, do it word by word and accept clashes and handle them
foreach (string KeyWord in KeyWordList)
{
if (!ExistingKeywordMatches.ContainsValue(KeyWord))
{
NewSearchDictionaryWordsList.Add(new SearchDictionary() { Word = KeyWord });
NewWordsAdded = true;
}
}
//algorithm: Attempt to add it to the db and get the id, if it fails with the expected exception for a duplicate word insertion attempt, then immediately read back that word and handle it
//Save the context in order to get the id's of the new words added
if (NewWordsAdded)
{
//adding in a range sped this up noticeably
ct.SearchDictionary.AddRange(NewSearchDictionaryWordsList);
try
{
ct.SaveChanges();
}
catch (Exception ex)
{
//In integration testing hit duplicate values of widget names
//adjusted uniquify code in test util class to use milliseconds instead of seconds (duh) and didn't hit this again,
//however just in case I'm leaving it here as a potential debug break point if it arises again
//Outside of a test situation I can't imagine this will be an issue in actual production use
//as it seems unlikely that two separate manual users inputting data are going to generate the same word at the same exact moment
//in time
throw ex;
}
}
//ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY
SearchDictionary NewWord = new SearchDictionary();
NewWord.Word = KeyWord;
//-----
//Now add the id's of the newly created words to the matching keyword id list for this object
foreach (SearchDictionary SD in ct.SearchDictionary.Local)
{
bool IsName = false;
if (NameKeyWordList.Contains(SD.Word))
IsName = true;
//See if it's already in the matching keywordlist or needs to be added
var ExistingMatch = MatchingKeywordIdList.Where(x => x.DictionaryId == SD.Id).FirstOrDefault();
if (ExistingMatch == null)//If null then needs to be added
MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = SD.Id, InName = IsName });
else
{
//Not null, but may need to be updated to reflect that it's in the name
if (!ExistingMatch.InName && IsName)
try
{
ExistingMatch.InName = true;
//ADD WORD TO DICTIONARY, SAVE THE ID INTO THE MATCHINGKEYWORDIDLIST
ct.SearchDictionary.Add(NewWord);
ct.SaveChanges();
MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = NewWord.Id, InName = NameKeyWordList.Contains(KeyWord) });
//It exists now
ExistingKeywordMatches.Add(NewWord.Id, NewWord.Word);
}
catch (Microsoft.EntityFrameworkCore.DbUpdateException ex)
{
//FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED?
if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx"))
{
throw ex;
}
//FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD
var SearchDictionaryMatchFoundInDB = ct.SearchDictionary.AsNoTracking().Where(x => x.Word == KeyWord).Single();
MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = SearchDictionaryMatchFoundInDB.Id, InName = NameKeyWordList.Contains(KeyWord) });
//It exists now
ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word);
}
catch(Exception ex){
var vv=ex.ToString();
}
}
}
#endregion second attempt
#region NEW WORD ADDITION: First naive attempt, clashing too much
// //ITERATE THROUGH THE KEYWORDS THAT DO *NOT* HAVE MATCHES IN THE SEARCHDICTIONARY AND ADD THEM TO THE SEARCH DICTIONARY, COLLECTING THEIR ID'S
// bool NewWordsAdded = false;
// var NewSearchDictionaryWordsList = new List<SearchDictionary>();
// foreach (string KeyWord in KeyWordList)
// {
// if (!ExistingKeywordMatches.ContainsValue(KeyWord))
// {
// NewSearchDictionaryWordsList.Add(new SearchDictionary() { Word = KeyWord });
// NewWordsAdded = true;
// }
// }
// //Save the context in order to get the id's of the new words added
// if (NewWordsAdded)
// {
// //adding in a range sped this up noticeably
// ct.SearchDictionary.AddRange(NewSearchDictionaryWordsList);
// try
// {
// ct.SaveChanges();
// }
// catch (Exception ex)
// {
// //In integration testing hit duplicate values of widget names
// //adjusted uniquify code in test util class to use milliseconds instead of seconds (duh) and didn't hit this again,
// //however just in case I'm leaving it here as a potential debug break point if it arises again
// //Outside of a test situation I can't imagine this will be an issue in actual production use
// //as it seems unlikely that two separate manual users inputting data are going to generate the same word at the same exact moment
// //in time
// //This is actually quite frequent and needs to be fixed, it can happen from widget or taggroup etc, need to handle this scenario far better
// //Rather than locking, maybe try a repeat method instead with a timeout
// /*
// 2018-10-10 02:48:38.2715|ERROR|Server Exception|Error=>Microsoft.EntityFrameworkCore.DbUpdateException: An error occurred while updating the entries. See the inner exception for details.
// ---> Npgsql.PostgresException: 23505: duplicate key value violates unique constraint "asearchdictionary_word_idx"
// */
// throw ex;
// }
// }
// //Now add the id's of the newly created words to the matching keyword id list for this object
// foreach (SearchDictionary SD in ct.SearchDictionary.Local)
// {
// bool IsName = false;
// if (NameKeyWordList.Contains(SD.Word))
// IsName = true;
// //See if it's already in the matching keywordlist or needs to be added
// var ExistingMatch = MatchingKeywordIdList.Where(x => x.DictionaryId == SD.Id).FirstOrDefault();
// if (ExistingMatch == null)//If null then needs to be added
// MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = SD.Id, InName = IsName });
// else
// {
// //Not null, but may need to be updated to reflect that it's in the name
// if (!ExistingMatch.InName && IsName)
// {
// ExistingMatch.InName = true;
// }
// }
// }
#endregion first naive attempt
//-------- END CRITICAL SECTION -------------
//-------------------------------------------
//CREATE THE SEARCHKEY RECORDS FOR ALL THE KEYWORDS
var NewSearchKeyList = new List<SearchKey>();
foreach (MatchingDictionaryEntry E in MatchingKeywordIdList)
{
NewSearchKeyList.Add(new SearchKey() { WordId = E.DictionaryId, InName = E.InName, ObjectId = objectID, ObjectType = objectType });
//ct.SearchKey.Add(new SearchKey() { WordId = E.DictionaryId, InName = E.InName, ObjectId = objectID, ObjectType = objectType });
}
ct.SearchKey.AddRange(NewSearchKeyList);
ct.SaveChanges();

3
server/AyaNova/util/AySchema.cs
View File

@@ -141,7 +141,10 @@ namespace AyaNova.Util
//SEARCH TABLES
exec("CREATE TABLE asearchdictionary (id BIGSERIAL PRIMARY KEY, word varchar(255) not null)");
//LOOKAT: this index is periodically being violated during testing
exec("CREATE UNIQUE INDEX asearchdictionary_word_idx ON asearchdictionary (word);");
exec("CREATE TABLE asearchkey (id BIGSERIAL PRIMARY KEY, wordid bigint not null REFERENCES asearchdictionary (id), objectid bigint not null, objecttype integer not null, inname bool not null)");
//create locale text tables

4
test/raven-integration/util.cs
View File

@@ -12,8 +12,8 @@ namespace raven_integration
{
private static HttpClient client { get; } = new HttpClient();
//private static string API_BASE_URL = "http://localhost:7575/api/v8.0/";
private static string API_BASE_URL = "https://test.helloayanova.com/api/v8.0/";
private static string API_BASE_URL = "http://localhost:7575/api/v8.0/";
//private static string API_BASE_URL = "https://test.helloayanova.com/api/v8.0/";
public static string TEST_DATA_FOLDER = @"..\..\..\testdata\";