From 6de6d43d0b999ce4014057767b4e65521253522b Mon Sep 17 00:00:00 2001 From: John Cardinal Date: Wed, 10 Oct 2018 18:40:32 +0000 Subject: [PATCH] --- server/AyaNova/biz/Search.cs | 158 ++++++++++++++++++++++---------- server/AyaNova/util/AySchema.cs | 3 + test/raven-integration/util.cs | 4 +- 3 files changed, 113 insertions(+), 52 deletions(-) diff --git a/server/AyaNova/biz/Search.cs b/server/AyaNova/biz/Search.cs index 7491a75f..1f18c58f 100644 --- a/server/AyaNova/biz/Search.cs +++ b/server/AyaNova/biz/Search.cs @@ -425,13 +425,12 @@ namespace AyaNova.Biz return; } + //BUILD A LIST OF MatchingDictionaryEntry items FOR THE MATCHING WORDS List MatchingKeywordIdList = new List(); - - //ITERATE ALL THE KEYWORDS, SEARCH IN THE SEARCHDICTIONARY TABLE AND COLLECT ID'S OF ANY PRE-EXISTING IN DB KEYWORDS - var ExistingKeywordMatches = ct.SearchDictionary.Where(m => KeyWordList.Contains(m.Word)).ToDictionary(m => m.Id, m => m.Word); + var ExistingKeywordMatches = ct.SearchDictionary.AsNoTracking().Where(m => KeyWordList.Contains(m.Word)).ToDictionary(m => m.Id, m => m.Word); //Put the matching keyword ID's into the list foreach (KeyValuePair K in ExistingKeywordMatches) { @@ -441,69 +440,128 @@ namespace AyaNova.Biz MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = K.Key, InName = IsName }); } - //ITERATE THROUGH THE KEYWORDS THAT DO *NOT* HAVE MATCHES IN THE SEARCHDICTIONARY AND ADD THEM TO THE SEARCH DICTIONARY, COLLECTING THEIR ID'S - bool NewWordsAdded = false; - var NewSearchDictionaryWordsList = new List(); + //-------- START CRITICAL SECTION ----------- + //------------------------------------------- + #region NEW WORD ADDITION second attempt, do it word by word and accept clashes and handle them foreach (string KeyWord in KeyWordList) { if (!ExistingKeywordMatches.ContainsValue(KeyWord)) { - NewSearchDictionaryWordsList.Add(new SearchDictionary() { Word = KeyWord }); - NewWordsAdded = true; - } - } + //algorithm: Attempt to add it to the db and get the id, if it fails with the expected exception for a duplicate word insertion attempt, then immediately read back that word and handle it - //Save the context in order to get the id's of the new words added - if (NewWordsAdded) - { - //adding in a range sped this up noticeably - ct.SearchDictionary.AddRange(NewSearchDictionaryWordsList); - try - { - ct.SaveChanges(); - } - catch (Exception ex) - { - //In integration testing hit duplicate values of widget names - //adjusted uniquify code in test util class to use milliseconds instead of seconds (duh) and didn't hit this again, - //however just in case I'm leaving it here as a potential debug break point if it arises again - //Outside of a test situation I can't imagine this will be an issue in actual production use - //as it seems unlikely that two separate manual users inputting data are going to generate the same word at the same exact moment - //in time - throw ex; - } - } + //ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY + SearchDictionary NewWord = new SearchDictionary(); + NewWord.Word = KeyWord; - - //----- - //Now add the id's of the newly created words to the matching keyword id list for this object - - foreach (SearchDictionary SD in ct.SearchDictionary.Local) - { - bool IsName = false; - if (NameKeyWordList.Contains(SD.Word)) - IsName = true; - //See if it's already in the matching keywordlist or needs to be added - var ExistingMatch = MatchingKeywordIdList.Where(x => x.DictionaryId == SD.Id).FirstOrDefault(); - - if (ExistingMatch == null)//If null then needs to be added - MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = SD.Id, InName = IsName }); - else - { - //Not null, but may need to be updated to reflect that it's in the name - if (!ExistingMatch.InName && IsName) + try { - ExistingMatch.InName = true; + //ADD WORD TO DICTIONARY, SAVE THE ID INTO THE MATCHINGKEYWORDIDLIST + ct.SearchDictionary.Add(NewWord); + ct.SaveChanges(); + MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = NewWord.Id, InName = NameKeyWordList.Contains(KeyWord) }); + //It exists now + ExistingKeywordMatches.Add(NewWord.Id, NewWord.Word); + } + catch (Microsoft.EntityFrameworkCore.DbUpdateException ex) + { + //FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED? + if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx")) + { + throw ex; + } + + //FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD + var SearchDictionaryMatchFoundInDB = ct.SearchDictionary.AsNoTracking().Where(x => x.Word == KeyWord).Single(); + MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = SearchDictionaryMatchFoundInDB.Id, InName = NameKeyWordList.Contains(KeyWord) }); + //It exists now + ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word); + } + catch(Exception ex){ + var vv=ex.ToString(); } } } + #endregion second attempt + + #region NEW WORD ADDITION: First naive attempt, clashing too much + + // //ITERATE THROUGH THE KEYWORDS THAT DO *NOT* HAVE MATCHES IN THE SEARCHDICTIONARY AND ADD THEM TO THE SEARCH DICTIONARY, COLLECTING THEIR ID'S + // bool NewWordsAdded = false; + // var NewSearchDictionaryWordsList = new List(); + // foreach (string KeyWord in KeyWordList) + // { + // if (!ExistingKeywordMatches.ContainsValue(KeyWord)) + // { + // NewSearchDictionaryWordsList.Add(new SearchDictionary() { Word = KeyWord }); + // NewWordsAdded = true; + // } + // } + + // //Save the context in order to get the id's of the new words added + // if (NewWordsAdded) + // { + // //adding in a range sped this up noticeably + // ct.SearchDictionary.AddRange(NewSearchDictionaryWordsList); + // try + // { + // ct.SaveChanges(); + // } + // catch (Exception ex) + // { + // //In integration testing hit duplicate values of widget names + // //adjusted uniquify code in test util class to use milliseconds instead of seconds (duh) and didn't hit this again, + // //however just in case I'm leaving it here as a potential debug break point if it arises again + // //Outside of a test situation I can't imagine this will be an issue in actual production use + // //as it seems unlikely that two separate manual users inputting data are going to generate the same word at the same exact moment + // //in time + + // //This is actually quite frequent and needs to be fixed, it can happen from widget or taggroup etc, need to handle this scenario far better + // //Rather than locking, maybe try a repeat method instead with a timeout + + // /* + // 2018-10-10 02:48:38.2715|ERROR|Server Exception|Error=>Microsoft.EntityFrameworkCore.DbUpdateException: An error occurred while updating the entries. See the inner exception for details. + // ---> Npgsql.PostgresException: 23505: duplicate key value violates unique constraint "asearchdictionary_word_idx" + // */ + // throw ex; + // } + // } + + + // //Now add the id's of the newly created words to the matching keyword id list for this object + + // foreach (SearchDictionary SD in ct.SearchDictionary.Local) + // { + // bool IsName = false; + // if (NameKeyWordList.Contains(SD.Word)) + // IsName = true; + // //See if it's already in the matching keywordlist or needs to be added + // var ExistingMatch = MatchingKeywordIdList.Where(x => x.DictionaryId == SD.Id).FirstOrDefault(); + + // if (ExistingMatch == null)//If null then needs to be added + // MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = SD.Id, InName = IsName }); + // else + // { + // //Not null, but may need to be updated to reflect that it's in the name + // if (!ExistingMatch.InName && IsName) + // { + // ExistingMatch.InName = true; + // } + // } + // } + + #endregion first naive attempt + + //-------- END CRITICAL SECTION ------------- + //------------------------------------------- + + + //CREATE THE SEARCHKEY RECORDS FOR ALL THE KEYWORDS var NewSearchKeyList = new List(); foreach (MatchingDictionaryEntry E in MatchingKeywordIdList) { NewSearchKeyList.Add(new SearchKey() { WordId = E.DictionaryId, InName = E.InName, ObjectId = objectID, ObjectType = objectType }); - //ct.SearchKey.Add(new SearchKey() { WordId = E.DictionaryId, InName = E.InName, ObjectId = objectID, ObjectType = objectType }); } ct.SearchKey.AddRange(NewSearchKeyList); ct.SaveChanges(); diff --git a/server/AyaNova/util/AySchema.cs b/server/AyaNova/util/AySchema.cs index a03ec8b7..1d95b7a0 100644 --- a/server/AyaNova/util/AySchema.cs +++ b/server/AyaNova/util/AySchema.cs @@ -141,7 +141,10 @@ namespace AyaNova.Util //SEARCH TABLES exec("CREATE TABLE asearchdictionary (id BIGSERIAL PRIMARY KEY, word varchar(255) not null)"); + + //LOOKAT: this index is periodically being violated during testing exec("CREATE UNIQUE INDEX asearchdictionary_word_idx ON asearchdictionary (word);"); + exec("CREATE TABLE asearchkey (id BIGSERIAL PRIMARY KEY, wordid bigint not null REFERENCES asearchdictionary (id), objectid bigint not null, objecttype integer not null, inname bool not null)"); //create locale text tables diff --git a/test/raven-integration/util.cs b/test/raven-integration/util.cs index 8d84ca9b..a3786fbd 100644 --- a/test/raven-integration/util.cs +++ b/test/raven-integration/util.cs @@ -12,8 +12,8 @@ namespace raven_integration { private static HttpClient client { get; } = new HttpClient(); - //private static string API_BASE_URL = "http://localhost:7575/api/v8.0/"; - private static string API_BASE_URL = "https://test.helloayanova.com/api/v8.0/"; + private static string API_BASE_URL = "http://localhost:7575/api/v8.0/"; + //private static string API_BASE_URL = "https://test.helloayanova.com/api/v8.0/"; public static string TEST_DATA_FOLDER = @"..\..\..\testdata\";