diff --git a/server/AyaNova/ControllerHelpers/ApiUploadProcessor.cs b/server/AyaNova/ControllerHelpers/ApiUploadProcessor.cs index c80498d6..cfff82d5 100644 --- a/server/AyaNova/ControllerHelpers/ApiUploadProcessor.cs +++ b/server/AyaNova/ControllerHelpers/ApiUploadProcessor.cs @@ -63,7 +63,7 @@ namespace AyaNova.Api.ControllerHelpers var CleanedUploadFileName = contentDisposition.FileName.Value.Replace("\"", ""); //get temp file path and temp file name - filePathAndName = FileUtil.NewRandomUserFilesFolderFileName; + filePathAndName = FileUtil.NewRandomAttachmentFilesFolderFileName; //save to disk using (var stream = new FileStream(filePathAndName, FileMode.Create)) diff --git a/server/AyaNova/Controllers/BackupController.cs b/server/AyaNova/Controllers/BackupController.cs index 00601061..a537c5a7 100644 --- a/server/AyaNova/Controllers/BackupController.cs +++ b/server/AyaNova/Controllers/BackupController.cs @@ -128,13 +128,13 @@ namespace AyaNova.Api.Controllers return StatusCode(403, new ApiNotAuthorizedResponse()); } - if (!FileUtil.UtilityFileExists(fileName)) + if (!FileUtil.BackupFileExists(fileName)) { await Task.Delay(nFailedAuthDelay);//fishing protection return NotFound(new ApiErrorResponse(ApiErrorCode.NOT_FOUND)); } string mimetype = fileName.EndsWith("zip") ? "application/zip" : "application/octet-stream"; - var utilityFilePath = FileUtil.GetFullPathForUtilityFile(fileName); + var utilityFilePath = FileUtil.GetFullPathForBackupFile(fileName); await EventLogProcessor.LogEventToDatabaseAsync(new Event(DownloadUser.Id, 0, AyaType.NoType, AyaEvent.UtilityFileDownload, fileName), ct); return PhysicalFile(utilityFilePath, mimetype, fileName); diff --git a/server/AyaNova/biz/AttachmentBiz.cs b/server/AyaNova/biz/AttachmentBiz.cs index 22bf7f03..cd618863 100644 --- a/server/AyaNova/biz/AttachmentBiz.cs +++ b/server/AyaNova/biz/AttachmentBiz.cs @@ -4,14 +4,8 @@ using AyaNova.Api.ControllerHelpers; using AyaNova.Models; using Microsoft.Extensions.Logging; using System; -using System.Threading.Tasks; -using System.Collections.Generic; -using AyaNova.Models; -using AyaNova.Biz; -using Microsoft.Extensions.Logging; -using Bogus; -using AyaNova.Api.ControllerHelpers; -using System.Diagnostics; +using Microsoft.EntityFrameworkCore; +using System.Linq; namespace AyaNova.Biz @@ -70,13 +64,39 @@ namespace AyaNova.Biz await JobsBiz.LogJobAsync(job.GId, $"Starting..."); apiServerState.SetOpsOnly("Attachment file maintenance"); - //JObject jobData = JObject.Parse(job.JobInfo); - // var seedLevel = (Seeder.Level.SeedLevel)jobData["seedLevel"].Value(); - // var timeZoneOffset = jobData["timeZoneOffset"].Value(); - // var seed = new Util.Seeder(); - // await seed.SeedDatabaseAsync(seedLevel, job.GId, timeZoneOffset); - // EXISTENCE CHECK + //iterate all records in chunks + bool moreRecords = true; + int skip = 0; + int chunkSize = 100; + do + { + var chunk = await ct.FileAttachment.AsNoTracking().OrderBy(z => z.Id).Skip(skip).Take(chunkSize).Select(z => new NameIdItem { Id = z.Id, Name = z.StoredFileName }).ToListAsync(); + if (chunk.Count < chunkSize) + { + //we've reached the end + moreRecords = false; + } + skip += chunkSize; + foreach (NameIdItem i in chunk) + { + //Does file exists where it's supposed to be? + if (!FileUtil.AttachmentFileExists(i.Name)) + { + var f = await ct.FileAttachment.FirstOrDefaultAsync(z => z.Id == i.Id); + if (f != null) + { + f.Exists = false; + await ct.SaveChangesAsync(); + } + } + } + + } while (moreRecords); + // long totalRecs = await ct.FileAttachment.LongCountAsync(); + + + // iterate FileAttachment records, if physically present then flag as such // make sure it has a short circuit that if there are NO files physically then all are out of sync // this scenario is someone moving a db and not moving physical files diff --git a/server/AyaNova/generator/CoreJobBackup.cs b/server/AyaNova/generator/CoreJobBackup.cs index 48864bf4..53dada97 100644 --- a/server/AyaNova/generator/CoreJobBackup.cs +++ b/server/AyaNova/generator/CoreJobBackup.cs @@ -57,7 +57,7 @@ namespace AyaNova.Biz var DBNameParameter = $"--dbname=postgresql://{PostgresConnectionString.Username}:{PostgresConnectionString.Password}@{PostgresConnectionString.Host}:{PostgresConnectionString.Port}/{PostgresConnectionString.Database}"; var DataBackupFile = $"{DemandFileNamePrepend}db-{FileUtil.GetSafeDateFileName()}.backup";//presentation issue so don't use UTC for this one - DataBackupFile = FileUtil.GetFullPathForUtilityFile(DataBackupFile); + DataBackupFile = FileUtil.GetFullPathForBackupFile(DataBackupFile); var BackupUtilityCommand = "pg_dump"; if (!string.IsNullOrWhiteSpace(ServerBootConfig.AYANOVA_BACKUP_PG_DUMP_PATH)) diff --git a/server/AyaNova/generator/CoreJobMetricsSnapshot.cs b/server/AyaNova/generator/CoreJobMetricsSnapshot.cs index 332ab31f..864529a9 100644 --- a/server/AyaNova/generator/CoreJobMetricsSnapshot.cs +++ b/server/AyaNova/generator/CoreJobMetricsSnapshot.cs @@ -104,7 +104,7 @@ namespace AyaNova.Biz var AttachmentFilesInfo = FileUtil.GetAttachmentFolderSizeInfo(); //Available space - var UtilityFilesAvailableSpace = FileUtil.UtilityFilesDriveAvailableSpace(); + var UtilityFilesAvailableSpace = FileUtil.BackupFilesDriveAvailableSpace(); var AttachmentFilesAvailableSpace = FileUtil.AttachmentFilesDriveAvailableSpace(); using (AyContext ct = ServiceProviderProvider.DBContext) diff --git a/server/AyaNova/util/DbUtil.cs b/server/AyaNova/util/DbUtil.cs index e9d97799..892f15e7 100644 --- a/server/AyaNova/util/DbUtil.cs +++ b/server/AyaNova/util/DbUtil.cs @@ -244,7 +244,7 @@ namespace AyaNova.Util } //final cleanup step is to erase user uploaded files - FileUtil.EraseEntireContentsOfUserFilesFolder(); + FileUtil.EraseEntireContentsOfAttachmentFilesFolder(); } @@ -354,7 +354,7 @@ namespace AyaNova.Util } //If we got here then it's safe to erase the attachment files - FileUtil.EraseEntireContentsOfUserFilesFolder(); + FileUtil.EraseEntireContentsOfAttachmentFilesFolder(); apiServerState.ResumePriorState(); diff --git a/server/AyaNova/util/FileUtil.cs b/server/AyaNova/util/FileUtil.cs index ea52351a..6d56e69c 100644 --- a/server/AyaNova/util/FileUtil.cs +++ b/server/AyaNova/util/FileUtil.cs @@ -61,16 +61,16 @@ namespace AyaNova.Util } #endregion folder ensurance - #region Utility file handling + #region Utility (BACKUP) file handling /// /// Get a path combining supplied file name and backup files folder /// /// - internal static string GetFullPathForUtilityFile(string fileName) + internal static string GetFullPathForBackupFile(string fileName) { - return Path.Combine(UtilityFilesFolder, fileName); + return Path.Combine(BackupFilesFolder, fileName); } @@ -78,7 +78,7 @@ namespace AyaNova.Util /// Get backup folder /// /// - internal static string UtilityFilesFolder + internal static string BackupFilesFolder { get { @@ -116,7 +116,7 @@ namespace AyaNova.Util BackupStatus statusReport = new BackupStatus(); try { - statusReport.AvailableFreeSpace = GetBytesReadable(new System.IO.DriveInfo(Path.GetPathRoot(UtilityFilesFolder)).AvailableFreeSpace); + statusReport.AvailableFreeSpace = GetBytesReadable(new System.IO.DriveInfo(Path.GetPathRoot(BackupFilesFolder)).AvailableFreeSpace); } catch (Exception ex) { @@ -125,7 +125,7 @@ namespace AyaNova.Util log.LogError(ex, "FileUtil::BackupStatusReport error getting available space"); } - var backupFiles = Directory.EnumerateFiles(UtilityFilesFolder, "*"); + var backupFiles = Directory.EnumerateFiles(BackupFilesFolder, "*"); foreach (string file in backupFiles.OrderByDescending(z => z)) { @@ -165,8 +165,8 @@ namespace AyaNova.Util internal static DateTime MostRecentAutomatedBackupFileDate() { DateTime LastBackup = DateTime.MinValue; - var BackupPath = UtilityFilesFolder; - foreach (string file in Directory.EnumerateFiles(UtilityFilesFolder, "db-*.backup")) + var BackupPath = BackupFilesFolder; + foreach (string file in Directory.EnumerateFiles(BackupFilesFolder, "db-*.backup")) { var ThisFileTime = File.GetCreationTimeUtc(file); if (ThisFileTime > LastBackup) @@ -183,12 +183,12 @@ namespace AyaNova.Util /// /// name of utility file /// duh! - internal static bool UtilityFileExists(string fileName) + internal static bool BackupFileExists(string fileName) { if (string.IsNullOrWhiteSpace(fileName)) return false; - var utilityFilePath = GetFullPathForUtilityFile(fileName); + var utilityFilePath = GetFullPathForBackupFile(fileName); return File.Exists(utilityFilePath); } @@ -196,9 +196,9 @@ namespace AyaNova.Util /// /// DANGER: Erases all Utility files including backups etc /// - internal static void EraseEntireContentsOfUtilityFilesFolder() + internal static void EraseEntireContentsOfBackupFilesFolder() { - System.IO.DirectoryInfo di = new DirectoryInfo(UtilityFilesFolder); + System.IO.DirectoryInfo di = new DirectoryInfo(BackupFilesFolder); foreach (FileInfo file in di.EnumerateFiles()) { file.Delete(); @@ -220,7 +220,7 @@ namespace AyaNova.Util if (keepCount < 1) keepCount = 1; //Database backups - var BackupFileList = Directory.EnumerateFiles(UtilityFilesFolder, "db-*.backup"); + var BackupFileList = Directory.EnumerateFiles(BackupFilesFolder, "db-*.backup"); if (BackupFileList.Count() > keepCount) { //sort, skip newest x (keepcount) delete the rest @@ -233,7 +233,7 @@ namespace AyaNova.Util } //Attachment backups - BackupFileList = Directory.EnumerateFiles(UtilityFilesFolder, "at-*.zip"); + BackupFileList = Directory.EnumerateFiles(BackupFilesFolder, "at-*.zip"); if (BackupFileList.Count() > keepCount) { //sort, skip newest x (keepcount) delete the rest @@ -247,11 +247,11 @@ namespace AyaNova.Util } - internal static long UtilityFilesDriveAvailableSpace() + internal static long BackupFilesDriveAvailableSpace() { try { - return new System.IO.DriveInfo(Path.GetPathRoot(UtilityFilesFolder)).AvailableFreeSpace; + return new System.IO.DriveInfo(Path.GetPathRoot(BackupFilesFolder)).AvailableFreeSpace; } catch (Exception ex) { @@ -275,7 +275,7 @@ namespace AyaNova.Util /// internal static List ZipGetUtilityFileEntries(string zipFileName) { - return ZipGetEntries(GetFullPathForUtilityFile(zipFileName)); + return ZipGetEntries(GetFullPathForBackupFile(zipFileName)); } /// @@ -306,7 +306,7 @@ namespace AyaNova.Util /// Get user folder /// /// - internal static string UserFilesFolder + internal static string AttachmentFilesFolder { get { @@ -331,11 +331,11 @@ namespace AyaNova.Util /// Get a random file name with path to attachments folder /// /// - internal static string NewRandomUserFilesFolderFileName + internal static string NewRandomAttachmentFilesFolderFileName { get { - return Path.Combine(UserFilesFolder, NewRandomFileName); + return Path.Combine(AttachmentFilesFolder, NewRandomFileName); } } @@ -402,7 +402,7 @@ namespace AyaNova.Util /// Path without the file internal static string GetPermanentAttachmentPath(string hash) { - return Path.Combine(UserFilesFolder, hash[0].ToString(), hash[1].ToString(), hash[2].ToString()); + return Path.Combine(AttachmentFilesFolder, hash[0].ToString(), hash[1].ToString(), hash[2].ToString()); } /// @@ -412,7 +412,7 @@ namespace AyaNova.Util /// internal static string GetPermanentAttachmentFilePath(string hash) { - return Path.Combine(UserFilesFolder, hash[0].ToString(), hash[1].ToString(), hash[2].ToString(), hash); + return Path.Combine(AttachmentFilesFolder, hash[0].ToString(), hash[1].ToString(), hash[2].ToString(), hash); } @@ -455,9 +455,9 @@ namespace AyaNova.Util /// /// DANGER: Erases all user files /// - internal static void EraseEntireContentsOfUserFilesFolder() + internal static void EraseEntireContentsOfAttachmentFilesFolder() { - System.IO.DirectoryInfo di = new DirectoryInfo(UserFilesFolder); + System.IO.DirectoryInfo di = new DirectoryInfo(AttachmentFilesFolder); foreach (FileInfo file in di.EnumerateFiles()) { file.Delete(); @@ -474,9 +474,9 @@ namespace AyaNova.Util try { var AttachmentsBackupFile = $"{demandFileNamePrepend}at-{FileUtil.GetSafeDateFileName()}.zip";//presentation issue so don't use UTC for this one - AttachmentsBackupFile = GetFullPathForUtilityFile(AttachmentsBackupFile); + AttachmentsBackupFile = GetFullPathForBackupFile(AttachmentsBackupFile); - System.IO.Compression.ZipFile.CreateFromDirectory(UserFilesFolder, AttachmentsBackupFile); + System.IO.Compression.ZipFile.CreateFromDirectory(AttachmentFilesFolder, AttachmentsBackupFile); } catch (Exception ex) { @@ -492,7 +492,7 @@ namespace AyaNova.Util { try { - return new System.IO.DriveInfo(Path.GetPathRoot(UserFilesFolder)).AvailableFreeSpace; + return new System.IO.DriveInfo(Path.GetPathRoot(AttachmentFilesFolder)).AvailableFreeSpace; } catch (Exception ex) { @@ -502,6 +502,27 @@ namespace AyaNova.Util return 0; } } + + internal static IEnumerable GetAllAttachmentFilePaths() + { + return Directory.EnumerateFiles(AttachmentFilesFolder, "*", SearchOption.AllDirectories); + } + + + /// + /// Confirm if a file exists in the attachment folder + /// + /// name of attachment file + /// duh! + internal static bool AttachmentFileExists(string fileName) + { + if (string.IsNullOrWhiteSpace(fileName)) + return false; + + // var utilityFilePath = GetFullPathForBackupFile(fileName); + return File.Exists(GetPermanentAttachmentFilePath(fileName)); + + } #endregion attachment stuff #region General utilities @@ -563,7 +584,7 @@ namespace AyaNova.Util /// internal static FolderSizeInfo GetAttachmentFolderSizeInfo() { - return GetDirectorySize(new DirectoryInfo(UserFilesFolder)); + return GetDirectorySize(new DirectoryInfo(AttachmentFilesFolder)); } /// @@ -572,7 +593,7 @@ namespace AyaNova.Util /// internal static FolderSizeInfo GetUtilityFolderSizeInfo() { - return GetDirectorySize(new DirectoryInfo(UtilityFilesFolder)); + return GetDirectorySize(new DirectoryInfo(BackupFilesFolder)); }