server/AyaNova/ControllerHelpers/ApiUploadProcessor.cs

@@ -63,7 +63,7 @@ namespace AyaNova.Api.ControllerHelpers
                         var CleanedUploadFileName = contentDisposition.FileName.Value.Replace("\"", "");
 
                         //get temp file path and temp file name
-                        filePathAndName = FileUtil.NewRandomUserFilesFolderFileName;
+                        filePathAndName = FileUtil.NewRandomAttachmentFilesFolderFileName;
 
                         //save to disk
                         using (var stream = new FileStream(filePathAndName, FileMode.Create))

server/AyaNova/Controllers/BackupController.cs

@@ -128,13 +128,13 @@ namespace AyaNova.Api.Controllers
                 return StatusCode(403, new ApiNotAuthorizedResponse());
             }
 
-            if (!FileUtil.UtilityFileExists(fileName))
+            if (!FileUtil.BackupFileExists(fileName))
             {
                 await Task.Delay(nFailedAuthDelay);//fishing protection
                 return NotFound(new ApiErrorResponse(ApiErrorCode.NOT_FOUND));
             }
             string mimetype = fileName.EndsWith("zip") ? "application/zip" : "application/octet-stream";
-            var utilityFilePath = FileUtil.GetFullPathForUtilityFile(fileName);
+            var utilityFilePath = FileUtil.GetFullPathForBackupFile(fileName);
             await EventLogProcessor.LogEventToDatabaseAsync(new Event(DownloadUser.Id, 0, AyaType.NoType, AyaEvent.UtilityFileDownload, fileName), ct);
             return PhysicalFile(utilityFilePath, mimetype, fileName);
 

server/AyaNova/biz/AttachmentBiz.cs

@@ -4,14 +4,8 @@ using AyaNova.Api.ControllerHelpers;
 using AyaNova.Models;
 using Microsoft.Extensions.Logging;
 using System;
-using System.Threading.Tasks;
-using System.Collections.Generic;
-using AyaNova.Models;
-using AyaNova.Biz;
-using Microsoft.Extensions.Logging;
-using Bogus;
-using AyaNova.Api.ControllerHelpers;
-using System.Diagnostics;
+using Microsoft.EntityFrameworkCore;
+using System.Linq;
 
 
 namespace AyaNova.Biz
@@ -70,13 +64,39 @@ namespace AyaNova.Biz
                 await JobsBiz.LogJobAsync(job.GId, $"Starting...");
                 apiServerState.SetOpsOnly("Attachment file maintenance");
 
-                //JObject jobData = JObject.Parse(job.JobInfo);
-                // var seedLevel = (Seeder.Level.SeedLevel)jobData["seedLevel"].Value<int>();
-                // var timeZoneOffset = jobData["timeZoneOffset"].Value<decimal>();
-                // var seed = new Util.Seeder();
-                // await seed.SeedDatabaseAsync(seedLevel, job.GId, timeZoneOffset);
-
                 // EXISTENCE CHECK
+                //iterate all records in chunks
+                bool moreRecords = true;
+                int skip = 0;
+                int chunkSize = 100;
+                do
+                {
+                    var chunk = await ct.FileAttachment.AsNoTracking().OrderBy(z => z.Id).Skip(skip).Take(chunkSize).Select(z => new NameIdItem { Id = z.Id, Name = z.StoredFileName }).ToListAsync();
+                    if (chunk.Count < chunkSize)
+                    {
+                        //we've reached the end
+                        moreRecords = false;
+                    }
+                    skip += chunkSize;
+                    foreach (NameIdItem i in chunk)
+                    {
+                        //Does file exists where it's supposed to be?
+                        if (!FileUtil.AttachmentFileExists(i.Name))
+                        {
+                            var f = await ct.FileAttachment.FirstOrDefaultAsync(z => z.Id == i.Id);
+                            if (f != null)
+                            {
+                                f.Exists = false;
+                                await ct.SaveChangesAsync();
+                            }
+                        }
+                    }
+
+                } while (moreRecords);
+               // long totalRecs = await ct.FileAttachment.LongCountAsync();
+
+
+
                 // 	iterate FileAttachment records, if physically present then flag as such
                 // 	make sure it has a short circuit that if there are NO files physically then all are out of sync
                 // 		this scenario is someone moving a db and not moving physical files

server/AyaNova/generator/CoreJobBackup.cs

@@ -57,7 +57,7 @@ namespace AyaNova.Biz
                 var DBNameParameter = $"--dbname=postgresql://{PostgresConnectionString.Username}:{PostgresConnectionString.Password}@{PostgresConnectionString.Host}:{PostgresConnectionString.Port}/{PostgresConnectionString.Database}";
 
                 var DataBackupFile = $"{DemandFileNamePrepend}db-{FileUtil.GetSafeDateFileName()}.backup";//presentation issue so don't use UTC for this one
-                DataBackupFile = FileUtil.GetFullPathForUtilityFile(DataBackupFile);
+                DataBackupFile = FileUtil.GetFullPathForBackupFile(DataBackupFile);
 
                 var BackupUtilityCommand = "pg_dump";
                 if (!string.IsNullOrWhiteSpace(ServerBootConfig.AYANOVA_BACKUP_PG_DUMP_PATH))

server/AyaNova/generator/CoreJobMetricsSnapshot.cs

@@ -104,7 +104,7 @@ namespace AyaNova.Biz
                 var AttachmentFilesInfo = FileUtil.GetAttachmentFolderSizeInfo();
 
                 //Available space
-                var UtilityFilesAvailableSpace = FileUtil.UtilityFilesDriveAvailableSpace();
+                var UtilityFilesAvailableSpace = FileUtil.BackupFilesDriveAvailableSpace();
                 var AttachmentFilesAvailableSpace = FileUtil.AttachmentFilesDriveAvailableSpace();
 
                 using (AyContext ct = ServiceProviderProvider.DBContext)

server/AyaNova/util/DbUtil.cs

@@ -244,7 +244,7 @@ namespace AyaNova.Util
             }
 
             //final cleanup step is to erase user uploaded files
-            FileUtil.EraseEntireContentsOfUserFilesFolder();
+            FileUtil.EraseEntireContentsOfAttachmentFilesFolder();
         }
 
 
@@ -354,7 +354,7 @@ namespace AyaNova.Util
             }
 
             //If we got here then it's safe to erase the attachment files
-            FileUtil.EraseEntireContentsOfUserFilesFolder();
+            FileUtil.EraseEntireContentsOfAttachmentFilesFolder();
 
             apiServerState.ResumePriorState();
 

server/AyaNova/util/FileUtil.cs

@@ -61,16 +61,16 @@ namespace AyaNova.Util
         }
         #endregion folder ensurance
 
-        #region Utility file handling
+        #region Utility (BACKUP) file handling
 
 
         /// <summary>
         /// Get a path combining supplied file name and backup files folder
         /// </summary>
         /// <returns></returns>
-        internal static string GetFullPathForUtilityFile(string fileName)
+        internal static string GetFullPathForBackupFile(string fileName)
         {
-            return Path.Combine(UtilityFilesFolder, fileName);
+            return Path.Combine(BackupFilesFolder, fileName);
         }
 
 
@@ -78,7 +78,7 @@ namespace AyaNova.Util
         /// Get backup folder
         /// </summary>
         /// <returns></returns>
-        internal static string UtilityFilesFolder
+        internal static string BackupFilesFolder
         {
             get
             {
@@ -116,7 +116,7 @@ namespace AyaNova.Util
             BackupStatus statusReport = new BackupStatus();
             try
             {
-                statusReport.AvailableFreeSpace = GetBytesReadable(new System.IO.DriveInfo(Path.GetPathRoot(UtilityFilesFolder)).AvailableFreeSpace);
+                statusReport.AvailableFreeSpace = GetBytesReadable(new System.IO.DriveInfo(Path.GetPathRoot(BackupFilesFolder)).AvailableFreeSpace);
             }
             catch (Exception ex)
             {
@@ -125,7 +125,7 @@ namespace AyaNova.Util
                 log.LogError(ex, "FileUtil::BackupStatusReport error getting available space");
             }
 
-            var backupFiles = Directory.EnumerateFiles(UtilityFilesFolder, "*");
+            var backupFiles = Directory.EnumerateFiles(BackupFilesFolder, "*");
 
             foreach (string file in backupFiles.OrderByDescending(z => z))
             {
@@ -165,8 +165,8 @@ namespace AyaNova.Util
         internal static DateTime MostRecentAutomatedBackupFileDate()
         {
             DateTime LastBackup = DateTime.MinValue;
-            var BackupPath = UtilityFilesFolder;
-            foreach (string file in Directory.EnumerateFiles(UtilityFilesFolder, "db-*.backup"))
+            var BackupPath = BackupFilesFolder;
+            foreach (string file in Directory.EnumerateFiles(BackupFilesFolder, "db-*.backup"))
             {
                 var ThisFileTime = File.GetCreationTimeUtc(file);
                 if (ThisFileTime > LastBackup)
@@ -183,12 +183,12 @@ namespace AyaNova.Util
         /// </summary>
         /// <param name="fileName">name of utility file </param>
         /// <returns>duh!</returns>
-        internal static bool UtilityFileExists(string fileName)
+        internal static bool BackupFileExists(string fileName)
         {
             if (string.IsNullOrWhiteSpace(fileName))
                 return false;
 
-            var utilityFilePath = GetFullPathForUtilityFile(fileName);
+            var utilityFilePath = GetFullPathForBackupFile(fileName);
             return File.Exists(utilityFilePath);
 
         }
@@ -196,9 +196,9 @@ namespace AyaNova.Util
         /// <summary>
         /// DANGER: Erases all Utility files including backups etc
         /// </summary>
-        internal static void EraseEntireContentsOfUtilityFilesFolder()
+        internal static void EraseEntireContentsOfBackupFilesFolder()
         {
-            System.IO.DirectoryInfo di = new DirectoryInfo(UtilityFilesFolder);
+            System.IO.DirectoryInfo di = new DirectoryInfo(BackupFilesFolder);
             foreach (FileInfo file in di.EnumerateFiles())
             {
                 file.Delete();
@@ -220,7 +220,7 @@ namespace AyaNova.Util
             if (keepCount < 1) keepCount = 1;
 
             //Database backups
-            var BackupFileList = Directory.EnumerateFiles(UtilityFilesFolder, "db-*.backup");
+            var BackupFileList = Directory.EnumerateFiles(BackupFilesFolder, "db-*.backup");
             if (BackupFileList.Count() > keepCount)
             {
                 //sort, skip newest x (keepcount) delete the rest
@@ -233,7 +233,7 @@ namespace AyaNova.Util
             }
 
             //Attachment backups
-            BackupFileList = Directory.EnumerateFiles(UtilityFilesFolder, "at-*.zip");
+            BackupFileList = Directory.EnumerateFiles(BackupFilesFolder, "at-*.zip");
             if (BackupFileList.Count() > keepCount)
             {
                 //sort, skip newest x (keepcount) delete the rest
@@ -247,11 +247,11 @@ namespace AyaNova.Util
 
         }
 
-        internal static long UtilityFilesDriveAvailableSpace()
+        internal static long BackupFilesDriveAvailableSpace()
         {
             try
             {
-                return new System.IO.DriveInfo(Path.GetPathRoot(UtilityFilesFolder)).AvailableFreeSpace;
+                return new System.IO.DriveInfo(Path.GetPathRoot(BackupFilesFolder)).AvailableFreeSpace;
             }
             catch (Exception ex)
             {
@@ -275,7 +275,7 @@ namespace AyaNova.Util
         /// <returns></returns>
         internal static List<string> ZipGetUtilityFileEntries(string zipFileName)
         {
-            return ZipGetEntries(GetFullPathForUtilityFile(zipFileName));
+            return ZipGetEntries(GetFullPathForBackupFile(zipFileName));
         }
 
         /// <summary>
@@ -306,7 +306,7 @@ namespace AyaNova.Util
         /// Get user folder
         /// </summary>
         /// <returns></returns>
-        internal static string UserFilesFolder
+        internal static string AttachmentFilesFolder
         {
             get
             {
@@ -331,11 +331,11 @@ namespace AyaNova.Util
         /// Get a random file name with path to attachments folder
         /// </summary>
         /// <returns></returns>
-        internal static string NewRandomUserFilesFolderFileName
+        internal static string NewRandomAttachmentFilesFolderFileName
         {
             get
             {
-                return Path.Combine(UserFilesFolder, NewRandomFileName);
+                return Path.Combine(AttachmentFilesFolder, NewRandomFileName);
             }
         }
 
@@ -402,7 +402,7 @@ namespace AyaNova.Util
         /// <returns>Path without the file</returns>
         internal static string GetPermanentAttachmentPath(string hash)
         {
-            return Path.Combine(UserFilesFolder, hash[0].ToString(), hash[1].ToString(), hash[2].ToString());
+            return Path.Combine(AttachmentFilesFolder, hash[0].ToString(), hash[1].ToString(), hash[2].ToString());
         }
 
         /// <summary>
@@ -412,7 +412,7 @@ namespace AyaNova.Util
         /// <returns></returns>
         internal static string GetPermanentAttachmentFilePath(string hash)
         {
-            return Path.Combine(UserFilesFolder, hash[0].ToString(), hash[1].ToString(), hash[2].ToString(), hash);
+            return Path.Combine(AttachmentFilesFolder, hash[0].ToString(), hash[1].ToString(), hash[2].ToString(), hash);
         }
 
 
@@ -455,9 +455,9 @@ namespace AyaNova.Util
         /// <summary>
         /// DANGER: Erases all user files
         /// </summary>
-        internal static void EraseEntireContentsOfUserFilesFolder()
+        internal static void EraseEntireContentsOfAttachmentFilesFolder()
         {
-            System.IO.DirectoryInfo di = new DirectoryInfo(UserFilesFolder);
+            System.IO.DirectoryInfo di = new DirectoryInfo(AttachmentFilesFolder);
             foreach (FileInfo file in di.EnumerateFiles())
             {
                 file.Delete();
@@ -474,9 +474,9 @@ namespace AyaNova.Util
             try
             {
                 var AttachmentsBackupFile = $"{demandFileNamePrepend}at-{FileUtil.GetSafeDateFileName()}.zip";//presentation issue so don't use UTC for this one
-                AttachmentsBackupFile = GetFullPathForUtilityFile(AttachmentsBackupFile);
+                AttachmentsBackupFile = GetFullPathForBackupFile(AttachmentsBackupFile);
 
-                System.IO.Compression.ZipFile.CreateFromDirectory(UserFilesFolder, AttachmentsBackupFile);
+                System.IO.Compression.ZipFile.CreateFromDirectory(AttachmentFilesFolder, AttachmentsBackupFile);
             }
             catch (Exception ex)
             {
@@ -492,7 +492,7 @@ namespace AyaNova.Util
         {
             try
             {
-                return new System.IO.DriveInfo(Path.GetPathRoot(UserFilesFolder)).AvailableFreeSpace;
+                return new System.IO.DriveInfo(Path.GetPathRoot(AttachmentFilesFolder)).AvailableFreeSpace;
             }
             catch (Exception ex)
             {
@@ -502,6 +502,27 @@ namespace AyaNova.Util
                 return 0;
             }
         }
+
+        internal static IEnumerable<string> GetAllAttachmentFilePaths()
+        {
+            return Directory.EnumerateFiles(AttachmentFilesFolder, "*", SearchOption.AllDirectories);
+        }
+
+
+        /// <summary>
+        /// Confirm if a file exists in the attachment folder
+        /// </summary>
+        /// <param name="fileName">name of attachment file </param>
+        /// <returns>duh!</returns>
+        internal static bool AttachmentFileExists(string fileName)
+        {
+            if (string.IsNullOrWhiteSpace(fileName))
+                return false;
+
+            // var utilityFilePath = GetFullPathForBackupFile(fileName);
+            return File.Exists(GetPermanentAttachmentFilePath(fileName));
+
+        }
         #endregion attachment stuff
 
         #region General utilities
@@ -563,7 +584,7 @@ namespace AyaNova.Util
         /// <returns></returns>
         internal static FolderSizeInfo GetAttachmentFolderSizeInfo()
         {
-            return GetDirectorySize(new DirectoryInfo(UserFilesFolder));
+            return GetDirectorySize(new DirectoryInfo(AttachmentFilesFolder));
         }
 
         /// <summary>
@@ -572,7 +593,7 @@ namespace AyaNova.Util
         /// <returns></returns>
         internal static FolderSizeInfo GetUtilityFolderSizeInfo()
         {
-            return GetDirectorySize(new DirectoryInfo(UtilityFilesFolder));
+            return GetDirectorySize(new DirectoryInfo(BackupFilesFolder));
         }