From 4c33a05e6d22967e49bd8e3216e392d4b7b69c26 Mon Sep 17 00:00:00 2001 From: John Cardinal Date: Sat, 27 Jun 2020 00:04:51 +0000 Subject: [PATCH] --- server/AyaNova/biz/AttachmentBiz.cs | 72 ++++++++++++++++++++++++----- server/AyaNova/util/FileUtil.cs | 28 +++++++++++ 2 files changed, 89 insertions(+), 11 deletions(-) diff --git a/server/AyaNova/biz/AttachmentBiz.cs b/server/AyaNova/biz/AttachmentBiz.cs index e0c90844..535df28b 100644 --- a/server/AyaNova/biz/AttachmentBiz.cs +++ b/server/AyaNova/biz/AttachmentBiz.cs @@ -67,8 +67,10 @@ namespace AyaNova.Biz //get a list of all attachment files currently on disk var AllAttachmentFilesOnDisk = FileUtil.GetAllAttachmentFilePaths(); + List AllDBFileFullPath = new List(); + // EXISTENCE CHECK //iterate all records in chunks, update the existence bool field if it's incorrect only bool moreRecords = true; @@ -98,7 +100,7 @@ namespace AyaNova.Biz f.Exists = FileExistsInReality; await ct.SaveChangesAsync(); } - } + } } } while (moreRecords); @@ -123,21 +125,69 @@ namespace AyaNova.Biz // ORPHANED FILES CHECK var FilesOnDiskNotInDb = AllAttachmentFilesOnDisk.Except(AllDBFileFullPath); + await JobsBiz.LogJobAsync(job.GId, $"Found {FilesOnDiskNotInDb.Count()} physical files not known to be existing Attachments, creating attachment records tied to 'NoType' so they show in UI"); + + // FOREIGN FILES placed in folder directly outside of attachment system + // user thinks they can just drop them in or accidentally copies them here + // Or, user renames a folder for some reason? + // This is a good reason not to delete them, because they can just un-rename them to fix it + // SWEEPER JOB + // I think it should delete them outright, but maybe that's a bad idea, not sure + // ID them to see if they *should* be one of ours by the file name I guess since it's the only identifying characteristic + // is it in the correct folder which is named based on it's hash? + // Is it X characters long (they all are or not?) + // Does it have an extension? None of ours have an extension + + //Vet the file and see if it has the characteristics of an attachment before re-attaching it, if not, compile into a list then log it and notify + //Attach any found into the NOTHING object type with id 0 so they will be represented in attachment list for being dealt with + List ForeignFilesNotLikelyAttachmentsFoundInAttachmentsFolder = new List(); foreach (string orphan in FilesOnDiskNotInDb) { - FileAttachment fa = new FileAttachment(); - fa.AttachToObjectId = 0; - fa.AttachToObjectType = AyaType.NoType; - fa.ContentType = "application/octet-stream";//most generic type, we don't know what it is - fa.DisplayFileName = "FOUND" + FileUtil.GetSafeDateFileName(); - fa.LastModified = DateTime.UtcNow; - fa.Notes = "Found in attachments folder not linked to an object"; - fa.StoredFileName = System.IO.Path.GetFileName(orphan); - await ct.FileAttachment.AddAsync(fa); - await ct.SaveChangesAsync(); + if (FileUtil.AppearsToBeAnOrphanedAttachment(orphan)) + { + FileAttachment fa = new FileAttachment(); + fa.AttachToObjectId = 0; + fa.AttachToObjectType = AyaType.NoType; + fa.ContentType = "application/octet-stream";//most generic type, we don't know what it is + fa.DisplayFileName = "FOUND" + FileUtil.GetSafeDateFileName(); + fa.LastModified = DateTime.UtcNow; + fa.Notes = "Found in attachments folder not linked to an object"; + fa.StoredFileName = System.IO.Path.GetFileName(orphan); + await ct.FileAttachment.AddAsync(fa); + await ct.SaveChangesAsync(); + } + else + { + log.LogDebug($"Foreign file found in attachments folder but doesn't appear to belong {orphan}"); + ForeignFilesNotLikelyAttachmentsFoundInAttachmentsFolder.Add(orphan); + } } + //ok, these files could be important and shouldn't be here so notify and log as much as possible + var ffcount = ForeignFilesNotLikelyAttachmentsFoundInAttachmentsFolder.Count; + if (ffcount > 0) + { + string msg = string.Empty; + if (ffcount > 25) + { + //we can list them all + msg = $"{ffcount} files found in attachments folder that don't appear to belong there:"; + } + else + { + msg = $"{ffcount} files found in attachments folder that don't appear to belong there, here are the first 25"; + } + + log.LogDebug(msg); + await JobsBiz.LogJobAsync(job.GId, msg); + //TODO: notify OPSNOTIFY + var outList = ForeignFilesNotLikelyAttachmentsFoundInAttachmentsFolder.Take(25).ToList(); + foreach (string s in outList) + { log.LogDebug(s); + await JobsBiz.LogJobAsync(job.GId, s); + } + } await JobsBiz.LogJobAsync(job.GId, "Finished."); await JobsBiz.UpdateJobStatusAsync(job.GId, JobStatus.Completed); diff --git a/server/AyaNova/util/FileUtil.cs b/server/AyaNova/util/FileUtil.cs index 9748f953..239cb292 100644 --- a/server/AyaNova/util/FileUtil.cs +++ b/server/AyaNova/util/FileUtil.cs @@ -522,6 +522,34 @@ namespace AyaNova.Util return File.Exists(GetPermanentAttachmentFilePath(fileName)); } + + internal static bool AppearsToBeAnOrphanedAttachment(string fullPathName) + { + // is it in the correct folder which is named based on it's hash? + // Is it X characters long (they all are or not?) Not sure though actually, maybe they are all 64 characters, maybe not + // Does it have an extension? None of ours have an extension + + + if (Path.HasExtension(fullPathName)) return false; + var FileName = Path.GetFileName(fullPathName); + //2339371F6C0C88656888163072635B282BB7FFF7B33771AB2295C868A0FECD34 + //3D67D4D258DCC7BB3CB560013C737E9865DFFB324C2012AA7E9E75CCCBE4133C + //BA7816BF8F01CFEA414140DE5DAE2223B00361A396177A9CB410FF61F20015AD + //40CE02D157C845E42AA4EF7DCC93A74B0179649C8D0A806B2F985D34AA7385CE + //9F2BA2DF87889B1E71346CC575A6F57334B441DB5AE4D40814F95E232C9539B5 + //got to be at least 32 chars + if (FileName.Length < 32) return false; + //probably all 64 chars but let's not count on that and go with folder is correct + + //what *should* the path be for a file of this name? + var ExpectedFullPath=GetPermanentAttachmentFilePath(FileName); + //if expected equals real then it's very likely an orphaned file + return fullPathName==ExpectedFullPath; + + + } + + #endregion attachment stuff #region General utilities