server/AyaNova/biz/Search.cs

@@ -425,13 +425,12 @@ namespace AyaNova.Biz
                 return;
             }
 
+
             //BUILD A LIST OF MatchingDictionaryEntry items FOR THE MATCHING WORDS
             List<MatchingDictionaryEntry> MatchingKeywordIdList = new List<MatchingDictionaryEntry>();
 
-
-
             //ITERATE ALL THE KEYWORDS, SEARCH IN THE SEARCHDICTIONARY TABLE AND COLLECT ID'S OF ANY PRE-EXISTING IN DB KEYWORDS
-            var ExistingKeywordMatches = ct.SearchDictionary.Where(m => KeyWordList.Contains(m.Word)).ToDictionary(m => m.Id, m => m.Word);
+            var ExistingKeywordMatches = ct.SearchDictionary.AsNoTracking().Where(m => KeyWordList.Contains(m.Word)).ToDictionary(m => m.Id, m => m.Word);
             //Put the matching keyword ID's into the list
             foreach (KeyValuePair<long, string> K in ExistingKeywordMatches)
             {
@@ -441,69 +440,128 @@ namespace AyaNova.Biz
                 MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = K.Key, InName = IsName });
             }
 
-            //ITERATE THROUGH THE KEYWORDS THAT DO *NOT* HAVE MATCHES IN THE SEARCHDICTIONARY AND ADD THEM TO THE SEARCH DICTIONARY, COLLECTING THEIR ID'S
+            //-------- START CRITICAL SECTION -----------
-            bool NewWordsAdded = false;
+            //-------------------------------------------            
-            var NewSearchDictionaryWordsList = new List<SearchDictionary>();
+            #region NEW WORD ADDITION second attempt, do it word by word and accept clashes and handle them
             foreach (string KeyWord in KeyWordList)
             {
                 if (!ExistingKeywordMatches.ContainsValue(KeyWord))
                 {
-                    NewSearchDictionaryWordsList.Add(new SearchDictionary() { Word = KeyWord });
+                    //algorithm: Attempt to add it to the db and get the id, if it fails with the expected exception for a duplicate word insertion attempt, then immediately read back that word and handle it
-                    NewWordsAdded = true;
-                }
-            }
 
-            //Save the context in order to get the id's of the new words added
+                    //ATTEMPT TO ADD THE WORD TO THE SEARCHDICTIONARY
-            if (NewWordsAdded)
+                    SearchDictionary NewWord = new SearchDictionary();
-            {
+                    NewWord.Word = KeyWord;
-                //adding in a range sped this up noticeably
-                ct.SearchDictionary.AddRange(NewSearchDictionaryWordsList);
-                try
-                {
-                    ct.SaveChanges();
-                }
-                catch (Exception ex)
-                {
-                    //In integration testing hit duplicate values of widget names
-                    //adjusted uniquify code in test util class to use milliseconds instead of seconds (duh) and didn't hit this again,
-                    //however just in case I'm leaving it here as a potential debug break point if it arises again
-                    //Outside of a test situation I can't imagine this will be an issue in actual production use
-                    //as it seems unlikely that two separate manual users inputting data are going to generate the same word at the same exact moment
-                    //in time
-                    throw ex;
-                }
-            }
 
-
+                    try
-            //-----
-            //Now add the id's of the newly created words to the matching keyword id list for this object
-
-            foreach (SearchDictionary SD in ct.SearchDictionary.Local)
-            {
-                bool IsName = false;
-                if (NameKeyWordList.Contains(SD.Word))
-                    IsName = true;
-                //See if it's already in the matching keywordlist or needs to be added
-                var ExistingMatch = MatchingKeywordIdList.Where(x => x.DictionaryId == SD.Id).FirstOrDefault();
-
-                if (ExistingMatch == null)//If null then needs to be added
-                    MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = SD.Id, InName = IsName });
-                else
-                {
-                    //Not null, but may need to be updated to reflect that it's in the name
-                    if (!ExistingMatch.InName && IsName)
                     {
-                        ExistingMatch.InName = true;
+                        //ADD WORD TO DICTIONARY, SAVE THE ID INTO THE MATCHINGKEYWORDIDLIST
+                        ct.SearchDictionary.Add(NewWord);
+                        ct.SaveChanges();
+                        MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = NewWord.Id, InName = NameKeyWordList.Contains(KeyWord) });
+                        //It exists now
+                        ExistingKeywordMatches.Add(NewWord.Id, NewWord.Word);
+                    }
+                    catch (Microsoft.EntityFrameworkCore.DbUpdateException ex)
+                    {
+                        //FAIL DUE TO OTHER CAUSE THAN WORD ALREADY ADDED?
+                        if (ex.InnerException == null || !ex.InnerException.Message.Contains("asearchdictionary_word_idx"))
+                        {
+                            throw ex;
+                        }
+
+                        //FETCH THE WORD ID, PLACE IN MATCHINGKEYWORDLIST AND MOVE ON TO THE NEXT WORD
+                        var SearchDictionaryMatchFoundInDB = ct.SearchDictionary.AsNoTracking().Where(x => x.Word == KeyWord).Single();
+                        MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = SearchDictionaryMatchFoundInDB.Id, InName = NameKeyWordList.Contains(KeyWord) });
+                        //It exists now
+                        ExistingKeywordMatches.Add(SearchDictionaryMatchFoundInDB.Id, SearchDictionaryMatchFoundInDB.Word);
+                    }
+                    catch(Exception ex){
+                        var vv=ex.ToString();
                     }
                 }
             }
 
+            #endregion second attempt
+
+            #region NEW WORD ADDITION: First naive attempt, clashing too much
+
+            // //ITERATE THROUGH THE KEYWORDS THAT DO *NOT* HAVE MATCHES IN THE SEARCHDICTIONARY AND ADD THEM TO THE SEARCH DICTIONARY, COLLECTING THEIR ID'S
+            // bool NewWordsAdded = false;
+            // var NewSearchDictionaryWordsList = new List<SearchDictionary>();
+            // foreach (string KeyWord in KeyWordList)
+            // {
+            //     if (!ExistingKeywordMatches.ContainsValue(KeyWord))
+            //     {
+            //         NewSearchDictionaryWordsList.Add(new SearchDictionary() { Word = KeyWord });
+            //         NewWordsAdded = true;
+            //     }
+            // }
+
+            // //Save the context in order to get the id's of the new words added
+            // if (NewWordsAdded)
+            // {
+            //     //adding in a range sped this up noticeably
+            //     ct.SearchDictionary.AddRange(NewSearchDictionaryWordsList);
+            //     try
+            //     {
+            //         ct.SaveChanges();
+            //     }
+            //     catch (Exception ex)
+            //     {
+            //         //In integration testing hit duplicate values of widget names
+            //         //adjusted uniquify code in test util class to use milliseconds instead of seconds (duh) and didn't hit this again,
+            //         //however just in case I'm leaving it here as a potential debug break point if it arises again
+            //         //Outside of a test situation I can't imagine this will be an issue in actual production use
+            //         //as it seems unlikely that two separate manual users inputting data are going to generate the same word at the same exact moment
+            //         //in time
+
+            //         //This is actually quite frequent and needs to be fixed, it can happen from widget or taggroup etc, need to handle this scenario far better
+            //         //Rather than locking, maybe try a repeat method instead with a timeout
+
+            //         /*
+            //         2018-10-10 02:48:38.2715|ERROR|Server Exception|Error=>Microsoft.EntityFrameworkCore.DbUpdateException: An error occurred while updating the entries. See the inner exception for details.
+            //              ---> Npgsql.PostgresException: 23505: duplicate key value violates unique constraint "asearchdictionary_word_idx"
+            //          */
+            //         throw ex;
+            //     }
+            // }
+
+
+            // //Now add the id's of the newly created words to the matching keyword id list for this object
+
+            // foreach (SearchDictionary SD in ct.SearchDictionary.Local)
+            // {
+            //     bool IsName = false;
+            //     if (NameKeyWordList.Contains(SD.Word))
+            //         IsName = true;
+            //     //See if it's already in the matching keywordlist or needs to be added
+            //     var ExistingMatch = MatchingKeywordIdList.Where(x => x.DictionaryId == SD.Id).FirstOrDefault();
+
+            //     if (ExistingMatch == null)//If null then needs to be added
+            //         MatchingKeywordIdList.Add(new MatchingDictionaryEntry() { DictionaryId = SD.Id, InName = IsName });
+            //     else
+            //     {
+            //         //Not null, but may need to be updated to reflect that it's in the name
+            //         if (!ExistingMatch.InName && IsName)
+            //         {
+            //             ExistingMatch.InName = true;
+            //         }
+            //     }
+            // }
+
+            #endregion first naive attempt
+
+            //-------- END CRITICAL SECTION -------------
+            //-------------------------------------------
+
+
+
             //CREATE THE SEARCHKEY RECORDS FOR ALL THE KEYWORDS 
             var NewSearchKeyList = new List<SearchKey>();
             foreach (MatchingDictionaryEntry E in MatchingKeywordIdList)
             {
                 NewSearchKeyList.Add(new SearchKey() { WordId = E.DictionaryId, InName = E.InName, ObjectId = objectID, ObjectType = objectType });
-                //ct.SearchKey.Add(new SearchKey() { WordId = E.DictionaryId, InName = E.InName, ObjectId = objectID, ObjectType = objectType });
             }
             ct.SearchKey.AddRange(NewSearchKeyList);
             ct.SaveChanges();

server/AyaNova/util/AySchema.cs

@@ -141,7 +141,10 @@ namespace AyaNova.Util
 
                 //SEARCH TABLES
                 exec("CREATE TABLE asearchdictionary (id BIGSERIAL PRIMARY KEY, word varchar(255) not null)");
+
+                //LOOKAT: this index is periodically being violated during testing
                 exec("CREATE UNIQUE INDEX asearchdictionary_word_idx ON asearchdictionary (word);");
+                
                 exec("CREATE TABLE asearchkey (id BIGSERIAL PRIMARY KEY, wordid bigint not null REFERENCES asearchdictionary (id), objectid bigint not null, objecttype integer not null, inname bool not null)");
 
                 //create locale text tables

test/raven-integration/util.cs

@@ -12,8 +12,8 @@ namespace raven_integration
     {
         private static HttpClient client { get; } = new HttpClient();
 
-        //private static string API_BASE_URL = "http://localhost:7575/api/v8.0/";
+        private static string API_BASE_URL = "http://localhost:7575/api/v8.0/";
-        private static string API_BASE_URL = "https://test.helloayanova.com/api/v8.0/";
+        //private static string API_BASE_URL = "https://test.helloayanova.com/api/v8.0/";
 
         public static string TEST_DATA_FOLDER = @"..\..\..\testdata\";