using System; using System.Linq; using System.Threading.Tasks; using System.IO; using System.IO.Compression; using System.Collections.Generic; using Microsoft.Extensions.Logging; using Microsoft.EntityFrameworkCore; using AyaNova.Models; using AyaNova.Biz; namespace AyaNova.Util { /* - Quickly generate large files in windows: http://tweaks.com/windows/62755/quickly-generate-large-test-files-in-windows/ */ internal static class FileUtil { #region Folder ensurance /// /// Ensurs folders exist and are not identical /// Throws an exception of they are found to be identical preventing startup /// The reason for this is to prevent a future erase database operation (which erases all attachment files) /// from erasing backups which might prevent recovery in case someone accidentally erases their database /// /// /// internal static void EnsureUserAndUtilityFoldersExistAndAreNotIdentical(string contentRootPath) { //UserFiles if (string.IsNullOrWhiteSpace(ServerBootConfig.AYANOVA_FOLDER_USER_FILES)) ServerBootConfig.AYANOVA_FOLDER_USER_FILES = Path.Combine(contentRootPath, "userfiles"); //BackupFiles if (ServerBootConfig.AYANOVA_FOLDER_BACKUP_FILES == null) ServerBootConfig.AYANOVA_FOLDER_BACKUP_FILES = Path.Combine(contentRootPath, "backupfiles"); //Temporary system files (reports etc) if (ServerBootConfig.AYANOVA_FOLDER_TEMPORARY_SERVER_FILES == null) ServerBootConfig.AYANOVA_FOLDER_TEMPORARY_SERVER_FILES = Path.Combine(contentRootPath, "tempfiles"); //Prevent using the same folder for both if ( string.Equals(Path.GetFullPath(ServerBootConfig.AYANOVA_FOLDER_USER_FILES), Path.GetFullPath(ServerBootConfig.AYANOVA_FOLDER_BACKUP_FILES), StringComparison.OrdinalIgnoreCase) || string.Equals(Path.GetFullPath(ServerBootConfig.AYANOVA_FOLDER_USER_FILES), Path.GetFullPath(ServerBootConfig.AYANOVA_FOLDER_TEMPORARY_SERVER_FILES), StringComparison.OrdinalIgnoreCase) || string.Equals(Path.GetFullPath(ServerBootConfig.AYANOVA_FOLDER_BACKUP_FILES), Path.GetFullPath(ServerBootConfig.AYANOVA_FOLDER_TEMPORARY_SERVER_FILES), StringComparison.OrdinalIgnoreCase) ) { throw new System.NotSupportedException("E1040: The configuration settings AYANOVA_FOLDER_USER_FILES, AYANOVA_FOLDER_BACKUP_FILES and AYANOVA_FOLDER_TEMPORARY_SYSTEM_FILES must all be different locations"); } EnsurePath(ServerBootConfig.AYANOVA_FOLDER_USER_FILES); EnsurePath(ServerBootConfig.AYANOVA_FOLDER_BACKUP_FILES); EnsurePath(ServerBootConfig.AYANOVA_FOLDER_TEMPORARY_SERVER_FILES); } //create path if doesn't exist already private static void EnsurePath(string path) { if (!Directory.Exists(path)) Directory.CreateDirectory(path); } #endregion folder ensurance #region Temporary files handling /// /// Get a path combining supplied file name and backup files folder /// /// internal static string GetFullPathForTemporaryFile(string fileName) { return Path.Combine(TemporaryFilesFolder, fileName); } /// /// Get backup file folder /// /// internal static string TemporaryFilesFolder { get { return ServerBootConfig.AYANOVA_FOLDER_TEMPORARY_SERVER_FILES; } } /// /// Get a random file name with path to temporary files folder /// /// internal static string NewRandomTempFilesFolderFileName { get { return Path.Combine(TemporaryFilesFolder, NewRandomFileName); } } /// /// Confirm if a file exists in the temporary files folder /// /// name of temp folder file /// duh! internal static bool TemporaryFileExists(string fileName) { if (string.IsNullOrWhiteSpace(fileName)) return false; var FilePath = GetFullPathForTemporaryFile(fileName); return File.Exists(FilePath); } /// /// Erase all files found to be older than age /// internal static void CleanTemporaryFilesFolder(TimeSpan age) { DateTime EraseIfOlderThan = DateTime.UtcNow - age; System.IO.DirectoryInfo di = new DirectoryInfo(TemporaryFilesFolder); foreach (FileInfo file in di.EnumerateFiles()) { if (file.CreationTimeUtc < EraseIfOlderThan) { file.Delete(); } } } #endregion #region Utility (BACKUP) file handling /// /// Get a path combining supplied file name and backup files folder /// /// internal static string GetFullPathForBackupFile(string fileName) { return Path.Combine(BackupFilesFolder, fileName); } /// /// Get backup folder /// /// internal static string BackupFilesFolder { get { return ServerBootConfig.AYANOVA_FOLDER_BACKUP_FILES; } } public class BackupFileInfo { public string length { get; set; } public string Name { get; set; } public DateTime Created { get; set; } } public class BackupStatus { public string AvailableFreeSpace { get; set; } public List BackupFiles { get; set; } public BackupStatus() { AvailableFreeSpace = null; BackupFiles = new List(); } } /// /// Get a status report of backup /// for reporting to ops user in UI /// /// internal static BackupStatus BackupStatusReport() { BackupStatus statusReport = new BackupStatus(); try { statusReport.AvailableFreeSpace = GetBytesReadable(new System.IO.DriveInfo(Path.GetPathRoot(BackupFilesFolder)).AvailableFreeSpace); } catch (Exception ex) { statusReport.AvailableFreeSpace = "ERROR"; ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("FileUtil::BackupStatus"); log.LogError(ex, "FileUtil::BackupStatusReport error getting available space"); } var backupFiles = Directory.EnumerateFiles(BackupFilesFolder, "*"); foreach (string file in backupFiles.OrderByDescending(z => z)) { var fi = new FileInfo(file); statusReport.BackupFiles.Add(new BackupFileInfo() { Name = Path.GetFileName(file), length = GetBytesReadable(fi.Length), Created = fi.CreationTimeUtc }); } return statusReport; } // List returnList = new List(); // foreach (string file in Directory.EnumerateFiles(UtilityFilesFolder, "*")) // { // var fi = new FileInfo(file); // returnList.Add(fi.Length); // } // returnList.Sort(); // return returnList; /// /// Get date of newest automatic backup file or minvalue if not found /// /// /// internal static DateTime MostRecentAutomatedBackupFileDate() { DateTime LastBackup = DateTime.MinValue; var BackupPath = BackupFilesFolder; foreach (string file in Directory.EnumerateFiles(BackupFilesFolder, "db-*.backup")) { var ThisFileTime = File.GetCreationTimeUtc(file); if (ThisFileTime > LastBackup) { LastBackup = ThisFileTime; } } return LastBackup; } /// /// Confirm if a file exists in the utility folder /// /// name of utility file /// duh! internal static bool BackupFileExists(string fileName) { if (string.IsNullOrWhiteSpace(fileName)) return false; var utilityFilePath = GetFullPathForBackupFile(fileName); return File.Exists(utilityFilePath); } /// /// DANGER: Erases all Utility files including backups etc /// internal static void EraseEntireContentsOfBackupFilesFolder() { System.IO.DirectoryInfo di = new DirectoryInfo(BackupFilesFolder); foreach (FileInfo file in di.EnumerateFiles()) { file.Delete(); } foreach (DirectoryInfo dir in di.EnumerateDirectories()) { dir.Delete(true); } } /// /// Cleanup excess backups (backup folder file) /// /// internal static void DatabaseBackupCleanUp(int keepCount) { if (keepCount < 1) keepCount = 1; //Database backups var BackupFileList = Directory.EnumerateFiles(BackupFilesFolder, "db-*.backup"); if (BackupFileList.Count() > keepCount) { //sort, skip newest x (keepcount) delete the rest var DeleteCount = BackupFileList.Count() - keepCount; var DeleteFileList = BackupFileList.OrderByDescending(m => m).Skip(keepCount).ToList(); foreach (string ExtraBackupFile in DeleteFileList) { File.Delete(ExtraBackupFile); } } //Attachment backups BackupFileList = Directory.EnumerateFiles(BackupFilesFolder, "at-*.zip"); if (BackupFileList.Count() > keepCount) { //sort, skip newest x (keepcount) delete the rest var DeleteCount = BackupFileList.Count() - keepCount; var DeleteFileList = BackupFileList.OrderByDescending(m => m).Skip(keepCount).ToList(); foreach (string ExtraBackupFile in DeleteFileList) { File.Delete(ExtraBackupFile); } } } internal static long BackupFilesDriveAvailableSpace() { try { return new System.IO.DriveInfo(Path.GetPathRoot(BackupFilesFolder)).AvailableFreeSpace; } catch (Exception ex) { ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("FileUtil::UtilityFilesDriveAvailableSpace"); log.LogError(ex, "FileUtil::UtilityFilesDriveAvailableSpace error getting available space"); return 0; } } #endregion Utility file handling #region Zip handling //////////////////////////////////////////////////////////////////////////////////////// //ZIP handling /// /// Get zip entries for a utlity file /// /// /// internal static List ZipGetUtilityFileEntries(string zipFileName) { return ZipGetEntries(GetFullPathForBackupFile(zipFileName)); } /// /// Get zip entries for full path and file name /// returns the entry fullname sorted alphabetically so that folders stay together /// /// /// internal static List ZipGetEntries(string zipPath) { List zipEntries = new List(); using (ZipArchive archive = ZipFile.OpenRead(zipPath)) { foreach (ZipArchiveEntry entry in archive.Entries) { zipEntries.Add(entry.FullName); } } zipEntries.Sort(); return zipEntries; } #endregion Zip handling #region Attachment file handling /// /// Get user folder /// /// internal static string AttachmentFilesFolder { get { return ServerBootConfig.AYANOVA_FOLDER_USER_FILES; } } /// /// Get a random file name with path to attachments folder /// /// internal static string NewRandomAttachmentFilesFolderFileName { get { return Path.Combine(AttachmentFilesFolder, NewRandomFileName); } } /// /// Store a file attachment /// /// internal static async Task StoreFileAttachmentAsync(string tempFilePath, string contentType, string fileName, DateTime lastModified, AyaTypeId attachToObject, string notes, AyContext ct) { //calculate hash var hash = FileHash.GetChecksum(tempFilePath); //Move to folder based on hash var permanentPath = GetPermanentAttachmentPath(hash); EnsurePath(permanentPath); var permanentFilePath = Path.Combine(permanentPath, hash); var FileSize = new FileInfo(tempFilePath).Length; //See if the file was already uploaded, if so then ignore it for now if (File.Exists(permanentFilePath)) { //delete the temp file, it's already stored File.Delete(tempFilePath); } else { System.IO.File.Move(tempFilePath, permanentFilePath); } //seems to be uploaded with the text null if (notes == "null") notes = string.Empty; //Build AyFileInfo FileAttachment fi = new FileAttachment() { StoredFileName = hash, DisplayFileName = fileName, Notes = notes, ContentType = contentType, AttachToObjectId = attachToObject.ObjectId, AttachToAType = attachToObject.AType, LastModified = lastModified, Size = FileSize }; //Store in DB await ct.FileAttachment.AddAsync(fi); await ct.SaveChangesAsync(); //Return AyFileInfo object return fi; } /// ///use first three characters for name of folders one character per folder, i.e.: ///if the checksum is f6a5b1236dbba1647257cc4646308326 ///it would be stored in userfiles/f/6/a/f6a5b1236dbba1647257cc4646308326 /// /// /// Path without the file internal static string GetPermanentAttachmentPath(string hash) { return Path.Combine(AttachmentFilesFolder, hash[0].ToString(), hash[1].ToString(), hash[2].ToString()); } /// /// Get the whole path including file name not just the folder /// /// /// internal static string GetPermanentAttachmentFilePath(string hash) { return Path.Combine(AttachmentFilesFolder, hash[0].ToString(), hash[1].ToString(), hash[2].ToString(), hash); } ////////////////////////////////////////////////////////// // // Delete all attachments for object // internal static async Task DeleteAttachmentsForObjectAsync(AyaType ayaType, long ayaId, AyContext ct) { var deleteList = await ct.FileAttachment.Where(z => z.AttachToObjectId == ayaId && z.AttachToAType == ayaType).ToListAsync(); foreach (var d in deleteList) { await DeleteFileAttachmentAsync(d, ct); } } /// /// Delete a file attachment /// checks ref count and if would be zero deletes file physically /// otherwise just deletes pointer in db /// /// /// /// internal static async Task DeleteFileAttachmentAsync(FileAttachment fileAttachmentToBeDeleted, AyContext ct) { //check ref count of file var count = await ct.FileAttachment.LongCountAsync(z => z.StoredFileName == fileAttachmentToBeDeleted.StoredFileName); //Remove from the DB ct.FileAttachment.Remove(fileAttachmentToBeDeleted); await ct.SaveChangesAsync(); if (count < 2) { //remove the file completely var permanentPath = GetPermanentAttachmentPath(fileAttachmentToBeDeleted.StoredFileName); var permanentFilePath = Path.Combine(permanentPath, fileAttachmentToBeDeleted.StoredFileName); if (File.Exists(permanentFilePath)) { //delete the temp file, it's already stored File.Delete(permanentFilePath); } } //Return AyFileInfo object return; } /// /// DANGER: Erases all user files /// internal static void EraseEntireContentsOfAttachmentFilesFolder() { System.IO.DirectoryInfo di = new DirectoryInfo(AttachmentFilesFolder); foreach (FileInfo file in di.EnumerateFiles()) { file.Delete(); } foreach (DirectoryInfo dir in di.EnumerateDirectories()) { dir.Delete(true); } } internal static void BackupAttachments(string demandFileNamePrepend, ILogger log = null) { try { var AttachmentsBackupFile = $"{demandFileNamePrepend}at-{FileUtil.GetSafeDateFileName()}.zip";//presentation issue so don't use UTC for this one AttachmentsBackupFile = GetFullPathForBackupFile(AttachmentsBackupFile); System.IO.Compression.ZipFile.CreateFromDirectory(AttachmentFilesFolder, AttachmentsBackupFile); } catch (Exception ex) { if (log != null) { log.LogError(ex, $"FileUtil::BackupAttachments"); } throw; } } internal static long AttachmentFilesDriveAvailableSpace() { try { return new System.IO.DriveInfo(Path.GetPathRoot(AttachmentFilesFolder)).AvailableFreeSpace; } catch (Exception ex) { ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("FileUtil::AttachmentFilesDriveAvailableSpace"); log.LogError(ex, "FileUtil::AttachmentFilesDriveAvailableSpace error getting available space"); return 0; } } internal static IEnumerable GetAllAttachmentFilePaths() { return Directory.EnumerateFiles(AttachmentFilesFolder, "*", SearchOption.AllDirectories); } /// /// Confirm if a file exists in the attachment folder /// /// name of attachment file internal static bool AttachmentFileExists(string fileName) { if (string.IsNullOrWhiteSpace(fileName)) return false; // var utilityFilePath = GetFullPathForBackupFile(fileName); return File.Exists(GetPermanentAttachmentFilePath(fileName)); } internal static bool AppearsToBeAnOrphanedAttachment(string fullPathName) { // is it in the correct folder which is named based on it's hash? // Is it X characters long (they all are or not?) Not sure though actually, maybe they are all 64 characters, maybe not // Does it have an extension? None of ours have an extension if (Path.HasExtension(fullPathName)) return false; var FileName = Path.GetFileName(fullPathName); //2339371F6C0C88656888163072635B282BB7FFF7B33771AB2295C868A0FECD34 //3D67D4D258DCC7BB3CB560013C737E9865DFFB324C2012AA7E9E75CCCBE4133C //BA7816BF8F01CFEA414140DE5DAE2223B00361A396177A9CB410FF61F20015AD //40CE02D157C845E42AA4EF7DCC93A74B0179649C8D0A806B2F985D34AA7385CE //9F2BA2DF87889B1E71346CC575A6F57334B441DB5AE4D40814F95E232C9539B5 //got to be at least 32 chars if (FileName.Length < 32) return false; //probably all 64 chars but let's not count on that and go with folder is correct //what *should* the path be for a file of this name? var ExpectedFullPath = GetPermanentAttachmentFilePath(FileName); //if expected equals real then it's very likely an orphaned file return fullPathName == ExpectedFullPath; } #endregion attachment stuff #region General utilities /// /// Get a random file name, no extension /// /// internal static string NewRandomFileName { get { return Path.GetRandomFileName(); } } //https://stackoverflow.com/a/11124118/8939 // Returns the human-readable file size for an arbitrary, 64-bit file size // The default format is "0.### XB", e.g. "4.2 KB" or "1.434 GB" public static string GetBytesReadable(long i) { // Get absolute value long absolute_i = (i < 0 ? -i : i); // Determine the suffix and readable value string suffix; double readable; if (absolute_i >= 0x1000000000000000) // Exabyte { suffix = "EiB"; readable = (i >> 50); } else if (absolute_i >= 0x4000000000000) // Petabyte { suffix = "PiB"; readable = (i >> 40); } else if (absolute_i >= 0x10000000000) // Terabyte { suffix = "TiB"; readable = (i >> 30); } else if (absolute_i >= 0x40000000) // Gigabyte { suffix = "GiB"; readable = (i >> 20); } else if (absolute_i >= 0x100000) // Megabyte { suffix = "MiB"; readable = (i >> 10); } else if (absolute_i >= 0x400) // Kilobyte { suffix = "KiB"; readable = i; } else { return i.ToString("0 B"); // Byte } // Divide by 1024 to get fractional value readable = (readable / 1024); // Return formatted number with suffix return readable.ToString("0.### ") + suffix; } /// /// Attachments / user files folder size info /// /// internal static FolderSizeInfo GetAttachmentFolderSizeInfo() { return GetDirectorySize(new DirectoryInfo(AttachmentFilesFolder)); } /// /// Utility / backup folder file size info /// /// internal static FolderSizeInfo GetBackupFolderSizeInfo() { return GetDirectorySize(new DirectoryInfo(BackupFilesFolder)); } /// /// Calculate disk space usage under . If is provided, /// then return subdirectory disk usages as well, up to levels deep. /// If levels is not provided or is 0, return a list with a single element representing the /// directory specified by . /// /// FROM https://stackoverflow.com/a/28094795/8939 /// /// /// public static FolderSizeInfo GetDirectorySize(DirectoryInfo root, int levels = 0) { var currentDirectory = new FolderSizeInfo(); // Add file sizes. FileInfo[] fis = root.GetFiles(); currentDirectory.Size = 0; foreach (FileInfo fi in fis) { currentDirectory.Size += fi.Length; } // Add subdirectory sizes. DirectoryInfo[] dis = root.GetDirectories(); currentDirectory.Path = root; currentDirectory.SizeWithChildren = currentDirectory.Size; currentDirectory.DirectoryCount = dis.Length; currentDirectory.DirectoryCountWithChildren = dis.Length; currentDirectory.FileCount = fis.Length; currentDirectory.FileCountWithChildren = fis.Length; if (levels >= 0) currentDirectory.Children = new List(); foreach (DirectoryInfo di in dis) { var dd = GetDirectorySize(di, levels - 1); if (levels >= 0) currentDirectory.Children.Add(dd); currentDirectory.SizeWithChildren += dd.SizeWithChildren; currentDirectory.DirectoryCountWithChildren += dd.DirectoryCountWithChildren; currentDirectory.FileCountWithChildren += dd.FileCountWithChildren; } return currentDirectory; } public class FolderSizeInfo { public DirectoryInfo Path { get; set; } public long SizeWithChildren { get; set; } public long Size { get; set; } public long DirectoryCount { get; set; } public long DirectoryCountWithChildren { get; set; } public long FileCount { get; set; } public long FileCountWithChildren { get; set; } public List Children { get; set; } } //Note assume local time because file times as this is used (backup etc) are a presentation issue not a db issue public static string GetSafeDateFileName() { return DateTime.Now.ToString("yyyyMMddHHmmssfff"); } public static string StringToSafeFileName(string fileName) {//https://stackoverflow.com/a/3678296/8939 if (string.IsNullOrWhiteSpace(fileName)) return "no_name"; char[] invalidFileNameChars = Path.GetInvalidFileNameChars(); // Builds a string out of valid chars and an _ for invalid ones var ret = new string(fileName.Select(ch => invalidFileNameChars.Contains(ch) ? '_' : ch).ToArray()); if (string.IsNullOrWhiteSpace(ret)) return "no_name"; return ret; } #endregion general utilities }//eoc }//eons