using System.Threading.Tasks; using Sockeye.Util; using Sockeye.Api.ControllerHelpers; using Sockeye.Models; using Microsoft.Extensions.Logging; using System; using Microsoft.EntityFrameworkCore; using System.Linq; using System.Collections.Generic; namespace Sockeye.Biz { /// /// Handle attachment file related cleanup and checking /// internal class AttachmentBiz : BizObject, IJobObject { internal AttachmentBiz(AyContext dbcontext, long currentUserId, AuthorizationRoles userRoles) { ct = dbcontext; UserId = currentUserId; CurrentUserRoles = userRoles; BizType = SockType.FileAttachment; } //////////////////////////////////////////////////////////////////////////////////////////////// //JOB / OPERATIONS // public async Task HandleJobAsync(OpsJob job) { //Hand off the particular job to the corresponding processing code //NOTE: If this code throws an exception the caller (JobsBiz::ProcessJobsAsync) will automatically set the job to failed and log the exeption so //basically any error condition during job processing should throw up an exception if it can't be handled //There might be future other job types so doing it like this for all biz job handlers for now switch (job.JobType) { case JobType.AttachmentMaintenance: await ProcessAttachmentMaintenanceAsync(job); break; default: throw new System.ArgumentOutOfRangeException($"AttachmentBiz.HandleJobAsync -> Invalid job type{job.JobType.ToString()}"); } } /// /// Handle the job /// /// private async Task ProcessAttachmentMaintenanceAsync(OpsJob job) { ILogger log = Sockeye.Util.ApplicationLogging.CreateLogger("AttachmentMaintenanceJob"); ApiServerState apiServerState = (ApiServerState)ServiceProviderProvider.Provider.GetService(typeof(ApiServerState)); //get the current server state so can set back to it later ApiServerState.ServerState wasServerState = apiServerState.GetState(); string wasReason = apiServerState.Reason; try { await JobsBiz.UpdateJobStatusAsync(job.GId, JobStatus.Running); await JobsBiz.LogJobAsync(job.GId, $"LT:StartJob "); apiServerState.SetOpsOnly("Attachment file maintenance"); //get a list of all attachment files currently on disk var AllAttachmentFilesOnDisk = FileUtil.GetAllAttachmentFilePaths(); List AllDBFileFullPath = new List(); // EXISTENCE CHECK //iterate all records in chunks, update the existence bool field if it's incorrect only bool moreRecords = true; int skip = 0; int chunkSize = 100; do { var chunk = await ct.FileAttachment.AsNoTracking().OrderBy(z => z.Id).Skip(skip).Take(chunkSize).ToListAsync(); if (chunk.Count < chunkSize) { //we've reached the end moreRecords = false; } skip += chunkSize; foreach (var i in chunk) { var FullPathName = FileUtil.GetPermanentAttachmentFilePath(i.StoredFileName); AllDBFileFullPath.Add(FullPathName); var FileExistsInReality = AllAttachmentFilesOnDisk.Contains(FullPathName); var ParentBizObjectExistsInReality = await BizObjectExistsInDatabase.ExistsAsync(i.AttachToAType, i.AttachToObjectId, ct); //does the db record reflect the same status as reality? if (FileExistsInReality != i.Exists || !ParentBizObjectExistsInReality) { var f = await ct.FileAttachment.FirstOrDefaultAsync(z => z.Id == i.Id); if (f != null) { f.Exists = FileExistsInReality; if (!ParentBizObjectExistsInReality) { //switch it to notype f.AttachToAType = SockType.NoType; f.AttachToObjectId = 0; } await ct.SaveChangesAsync(); } } //DOES THE PARENT OBJECT EXIST? } } while (moreRecords); //I kept this block because I did a lot of work to figure it out but in the end I don't need it because //a user will be moving attachments so they would no longer be existing on their old NOTYPE orphan location anyway // //DE-ORPHANIZE ACTION (clean up former orphans) // //people can attach an orphan to another record so this cleans that up // //also, potentiallly // //iterate orphaned file attachments to NOTHING type, if found to be attached to any other object remove the orphaned object attachment record in db // //but keeping the physical file since it's attached to something else // var AllOrphansInDb = await ct.FileAttachment.Where(z => z.AttachToAType == SockType.NoType).ToListAsync(); // foreach (FileAttachment OrphanInDb in AllOrphansInDb) // { // if (await ct.FileAttachment.AnyAsync(z => z.StoredFileName==OrphanInDb.StoredFileName && z.AttachToAType != SockType.NoType)) // { // //It is also attached to something else so remove it from the nothing type // ct.FileAttachment.Remove(OrphanInDb); // await ct.SaveChangesAsync(); // } // } // ORPHANED FILES CHECK var FilesOnDiskNotInDb = AllAttachmentFilesOnDisk.Except(AllDBFileFullPath); if (FilesOnDiskNotInDb.Count() > 0) await JobsBiz.LogJobAsync(job.GId, $"Found {FilesOnDiskNotInDb.Count()} physical files not known to be existing Attachments, creating attachment records tied to 'NoType' so they show in UI"); // FOREIGN FILES placed in folder directly outside of attachment system // user thinks they can just drop them in or accidentally copies them here // Or, user renames a folder for some reason? // This is a good reason not to delete them, because they can just un-rename them to fix it // SWEEPER JOB // I think it should delete them outright, but maybe that's a bad idea, not sure // ID them to see if they *should* be one of ours by the file name I guess since it's the only identifying characteristic // is it in the correct folder which is named based on it's hash? // Is it X characters long (they all are or not?) // Does it have an extension? None of ours have an extension //Vet the file and see if it has the characteristics of an attachment before re-attaching it, if not, compile into a list then log it and notify //Attach any found into the NOTHING object type with id 0 so they will be represented in attachment list for being dealt with List ForeignFilesNotLikelyAttachmentsFoundInAttachmentsFolder = new List(); foreach (string orphan in FilesOnDiskNotInDb) { if (FileUtil.AppearsToBeAnOrphanedAttachment(orphan)) { FileAttachment fa = new FileAttachment(); fa.AttachToObjectId = 0; fa.AttachToAType = SockType.NoType; fa.ContentType = "application/octet-stream";//most generic type, we don't know what it is fa.DisplayFileName = "FOUND" + FileUtil.GetSafeDateFileName(); fa.LastModified = DateTime.UtcNow; fa.Notes = "Found in attachments folder not linked to an object"; fa.StoredFileName = System.IO.Path.GetFileName(orphan); await ct.FileAttachment.AddAsync(fa); await ct.SaveChangesAsync(); } else { log.LogDebug($"Foreign file found in attachments folder but doesn't appear to belong {orphan}"); ForeignFilesNotLikelyAttachmentsFoundInAttachmentsFolder.Add(orphan); } } //ok, these files could be important and shouldn't be here so notify and log as much as possible var ffcount = ForeignFilesNotLikelyAttachmentsFoundInAttachmentsFolder.Count; if (ffcount > 0) { string msg = string.Empty; if (ffcount < 25) { //we can list them all msg = $"{ffcount} files found in attachments folder that don't appear to belong there:"; } else { msg = $"{ffcount} files found in attachments folder that don't appear to belong there, here are the first 25"; } log.LogDebug(msg); await JobsBiz.LogJobAsync(job.GId, msg); await NotifyEventHelper.AddOpsProblemEvent($"Attachments issue:{msg}"); var outList = ForeignFilesNotLikelyAttachmentsFoundInAttachmentsFolder.Take(25).ToList(); foreach (string s in outList) { log.LogDebug(s); await JobsBiz.LogJobAsync(job.GId, s); } } await JobsBiz.LogJobAsync(job.GId, "Finished."); await JobsBiz.UpdateJobStatusAsync(job.GId, JobStatus.Completed); } catch (Exception ex) { log.LogError(ex, "AttachmentMaintenanceJob error during ops"); await JobsBiz.LogJobAsync(job.GId, $"AttachmentMaintenanceJob error during ops\r\n{ex.Message}"); throw; } finally { log.LogInformation($"AttachmentMaintenanceJob: setting server state back to {wasServerState.ToString()}"); apiServerState.SetState(wasServerState, wasReason); } } //Other job handlers here... //////////////////////////////////////////////////////////////////////////////////////////////// //DUPLICATE ATTACHMENTS TO NEW OBJECT // internal static async Task DuplicateAttachments(SockTypeId aSource, SockTypeId aDest, AyContext ct) { var sources = await ct.FileAttachment.AsNoTracking() .Where(z => z.AttachToAType == aSource.SockType && z.AttachToObjectId == aSource.ObjectId) .ToListAsync(); if (sources.Count > 0) { foreach (var src in sources) { ct.FileAttachment.Add(new FileAttachment { AttachToObjectId = aDest.ObjectId, AttachToAType = aDest.SockType, StoredFileName = src.StoredFileName, DisplayFileName = src.DisplayFileName, ContentType = src.ContentType, LastModified = src.LastModified, Notes = src.Notes, Exists = src.Exists, Size = src.Size }); } await ct.SaveChangesAsync(); } } ///////////////////////////////////////////////////////////////////// }//eoc }//eons