using System; using System.Linq; using System.Threading.Tasks; using System.IO; using System.IO.Compression; using System.Collections.Generic; using Microsoft.Extensions.Logging; using Microsoft.EntityFrameworkCore; using AyaNova.Models; using AyaNova.Biz; using System.Reflection; namespace AyaNova.Util { /* - Quickly generate large files in windows: http://tweaks.com/windows/62755/quickly-generate-large-test-files-in-windows/ */ internal static class FileUtil { #region Folder ensurance /// /// Ensurs folders exist and are not identical /// Throws an exception of they are found to be identical preventing startup /// The reason for this is to prevent a future erase database operation (which erases all attachment files) /// from erasing backups which might prevent recovery in case someone accidentally erases their database /// /// internal static void EnsureUserAndUtilityFoldersExistAndAreNotIdentical() { // //UserFiles // if (string.IsNullOrWhiteSpace(ServerBootConfig.AYANOVA_ATTACHMENT_FILES_PATH)) // ServerBootConfig.AYANOVA_ATTACHMENT_FILES_PATH = Path.Combine(contentRootPath, "userfiles"); // //BackupFiles // if (ServerBootConfig.AYANOVA_BACKUP_FILES_PATH == null) // ServerBootConfig.AYANOVA_BACKUP_FILES_PATH = Path.Combine(contentRootPath, "backupfiles"); // //Temporary system files (reports etc) // if (ServerBootConfig.AYANOVA_TEMP_FILES_PATH == null) // ServerBootConfig.AYANOVA_TEMP_FILES_PATH = Path.Combine(contentRootPath, "tempfiles"); //Prevent using the same folder for both if ( string.Equals(Path.GetFullPath(ServerBootConfig.AYANOVA_ATTACHMENT_FILES_PATH), Path.GetFullPath(ServerBootConfig.AYANOVA_BACKUP_FILES_PATH), StringComparison.OrdinalIgnoreCase) || string.Equals(Path.GetFullPath(ServerBootConfig.AYANOVA_ATTACHMENT_FILES_PATH), Path.GetFullPath(ServerBootConfig.AYANOVA_TEMP_FILES_PATH), StringComparison.OrdinalIgnoreCase) || string.Equals(Path.GetFullPath(ServerBootConfig.AYANOVA_BACKUP_FILES_PATH), Path.GetFullPath(ServerBootConfig.AYANOVA_TEMP_FILES_PATH), StringComparison.OrdinalIgnoreCase) ) { throw new System.NotSupportedException("E1040: The configuration settings AYANOVA_ATTACHMENT_FILES_PATH, AYANOVA_BACKUP_FILES_PATH and AYANOVA_FOLDER_TEMPORARY_SYSTEM_FILES must all be different locations"); } EnsurePath(ServerBootConfig.AYANOVA_ATTACHMENT_FILES_PATH); EnsurePath(ServerBootConfig.AYANOVA_BACKUP_FILES_PATH); EnsurePath(ServerBootConfig.AYANOVA_TEMP_FILES_PATH); } //create path if doesn't exist already private static void EnsurePath(string path) { //Console.WriteLine($"FileUtil::EnsurePath path = [{path}]"); if (!Directory.Exists(path)) Directory.CreateDirectory(path); } #endregion folder ensurance #region Temporary files handling /// /// Get a path combining supplied file name and backup files folder /// /// internal static string GetFullPathForTemporaryFile(string fileName) { return Path.Combine(TemporaryFilesFolder, fileName); } /// /// Get backup file folder /// /// internal static string TemporaryFilesFolder { get { return ServerBootConfig.AYANOVA_TEMP_FILES_PATH; } } /// /// Get a random file name with path to temporary files folder /// /// internal static string NewRandomTempFilesFolderFileName { get { return Path.Combine(TemporaryFilesFolder, NewRandomFileName); } } /// /// Confirm if a file exists in the temporary files folder /// /// name of temp folder file /// duh! internal static bool TemporaryFileExists(string fileName) { if (string.IsNullOrWhiteSpace(fileName)) return false; var FilePath = GetFullPathForTemporaryFile(fileName); return File.Exists(FilePath); } /// /// Erase all files found to be older than age /// internal static void CleanTemporaryFilesFolder(TimeSpan age) { DateTime EraseIfOlderThan = DateTime.UtcNow - age; System.IO.DirectoryInfo di = new DirectoryInfo(TemporaryFilesFolder); foreach (FileInfo file in di.EnumerateFiles()) { if (file.CreationTimeUtc < EraseIfOlderThan) { /* 2022-03-08 14:42:13.8155|ERROR|JobsBiz|Server::ProcessJobsAsync unexpected error during processing|System.IO.IOException: The process cannot access the file 'c:\temp\ravendata\temp\vrimbqp2lia.pdf' because it is being used by another process. at System.IO.FileSystem.DeleteFile(String fullPath) at System.IO.FileInfo.Delete() at AyaNova.Util.FileUtil.CleanTemporaryFilesFolder(TimeSpan age) in C:\data\code\raven\server\AyaNova\util\FileUtil.cs:line 137 at AyaNova.Biz.CoreJobTempFolderCleanup.DoWork() in C:\data\code\raven\server\AyaNova\generator\CoreJobTempFolderCleanup.cs:line 42 at AyaNova.Biz.JobsBiz.ProcessJobsAsync() in C:\data\code\raven\server\AyaNova\biz\JobsBiz.cs:line 232 */ file.Delete(); } } } #endregion #region Utility (BACKUP) file handling /// /// Get a path combining supplied file name and backup files folder /// /// internal static string GetFullPathForBackupFile(string fileName) { return Path.Combine(ServerBootConfig.AYANOVA_BACKUP_FILES_PATH, fileName); } // /// // /// Get backup folder // /// // /// // internal static string BackupFilesFolder // { // get // { // return ServerBootConfig.AYANOVA_BACKUP_FILES_PATH; // } // } public class BackupFileInfo { public string length { get; set; } public string Name { get; set; } public DateTime Created { get; set; } } public class BackupStatus { public string AvailableFreeSpace { get; set; } public List BackupFiles { get; set; } public BackupStatus() { AvailableFreeSpace = null; BackupFiles = new List(); } } /// /// Get a status report of backup /// for reporting to ops user in UI /// /// internal static BackupStatus BackupStatusReport() { BackupStatus statusReport = new BackupStatus(); try { statusReport.AvailableFreeSpace = GetBytesReadable(new System.IO.DriveInfo(Path.GetPathRoot(ServerBootConfig.AYANOVA_BACKUP_FILES_PATH)).AvailableFreeSpace); } catch (Exception ex) { statusReport.AvailableFreeSpace = "ERROR"; ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("FileUtil::BackupStatus"); log.LogError(ex, "FileUtil::BackupStatusReport error getting available space"); } var backupFiles = Directory.EnumerateFiles(ServerBootConfig.AYANOVA_BACKUP_FILES_PATH, "*"); foreach (string file in backupFiles.OrderByDescending(z => z)) { var fi = new FileInfo(file); statusReport.BackupFiles.Add(new BackupFileInfo() { Name = Path.GetFileName(file), length = GetBytesReadable(fi.Length), Created = fi.CreationTimeUtc }); } return statusReport; } // List returnList = new List(); // foreach (string file in Directory.EnumerateFiles(UtilityFilesFolder, "*")) // { // var fi = new FileInfo(file); // returnList.Add(fi.Length); // } // returnList.Sort(); // return returnList; /// /// Get date of newest automatic backup file or minvalue if not found /// /// /// internal static DateTime MostRecentAutomatedBackupFileDate() { DateTime LastBackup = DateTime.MinValue; foreach (string file in Directory.EnumerateFiles(ServerBootConfig.AYANOVA_BACKUP_FILES_PATH, "db-*.backup")) { var ThisFileTime = File.GetCreationTimeUtc(file); if (ThisFileTime > LastBackup) { LastBackup = ThisFileTime; } } return LastBackup; } /// /// Confirm if a file exists in the utility folder /// /// name of utility file /// duh! internal static bool BackupFileExists(string fileName) { if (string.IsNullOrWhiteSpace(fileName)) return false; var utilityFilePath = GetFullPathForBackupFile(fileName); return File.Exists(utilityFilePath); } // /// // /// DANGER: Erases all Utility files including backups etc // /// // internal static void EraseEntireContentsOfBackupFilesFolder() // { // System.IO.DirectoryInfo di = new DirectoryInfo(BackupFilesFolder); // foreach (FileInfo file in di.EnumerateFiles()) // { // file.Delete(); // } // foreach (DirectoryInfo dir in di.EnumerateDirectories()) // { // dir.Delete(true); // } // } /// /// DANGER: Erase backup file if it exists by name /// internal static void EraseBackupFile(string name) { name = Path.GetFileName(name);//ensure no directory shenanigans, only a file name is allowed //remove the file completely var DeleteFilePath = Path.Combine(ServerBootConfig.AYANOVA_BACKUP_FILES_PATH, name); if (File.Exists(DeleteFilePath)) { //delete the temp file, it's already stored File.Delete(DeleteFilePath); } //never return an error as that would leak info if someone is fishing for deleting files that don't exist } /// /// Cleanup excess backups (backup folder file) /// /// internal static void DatabaseBackupCleanUp(int keepCount) { if (keepCount < 1) keepCount = 1; //case 4460 rename manual- file pattern to support removal of manual- prepend on demand backup //this ensures that pruning happens properly var renameList = Directory.EnumerateFiles(ServerBootConfig.AYANOVA_BACKUP_FILES_PATH, "manual-*"); if (renameList.Count() > 0) { foreach (string renameFile in renameList) { File.Move(renameFile, renameFile.Replace("manual-", ""), true); } } //Database backups var BackupFileList = Directory.EnumerateFiles(ServerBootConfig.AYANOVA_BACKUP_FILES_PATH, "*db-*.backup"); if (BackupFileList.Count() > keepCount) { //sort, skip newest x (keepcount) delete the rest var DeleteCount = BackupFileList.Count() - keepCount; var DeleteFileList = BackupFileList.OrderByDescending(m => m).Skip(keepCount).ToList(); foreach (string ExtraBackupFile in DeleteFileList) { File.Delete(ExtraBackupFile); } } //Attachment backups BackupFileList = Directory.EnumerateFiles(ServerBootConfig.AYANOVA_BACKUP_FILES_PATH, "*at-*.zip"); if (BackupFileList.Count() > keepCount) { //sort, skip newest x (keepcount) delete the rest var DeleteCount = BackupFileList.Count() - keepCount; var DeleteFileList = BackupFileList.OrderByDescending(m => m).Skip(keepCount).ToList(); foreach (string ExtraBackupFile in DeleteFileList) { File.Delete(ExtraBackupFile); } } } internal static long BackupFilesDriveAvailableSpace() { //Console.WriteLine("b fileutil:backupfilesdriveavailablespace, backupfilesfolder:", ServerBootConfig.AYANOVA_BACKUP_FILES_PATH); //Console.WriteLine("fileutil:backupfilesdriveavailablespace, backupfilesfolder FULLPATH:", Path.GetFullPath(ServerBootConfig.AYANOVA_BACKUP_FILES_PATH)); // Console.WriteLine("fileutil:backupfilesdriveavailablespace, backupfilesfolder PATHROOT:", Path.GetPathRoot(ServerBootConfig.AYANOVA_BACKUP_FILES_PATH)); return new System.IO.DriveInfo(Path.GetPathRoot(ServerBootConfig.AYANOVA_BACKUP_FILES_PATH)).AvailableFreeSpace; } #endregion Utility file handling #region Zip handling //////////////////////////////////////////////////////////////////////////////////////// //ZIP handling /// /// Get zip entries for a utlity file /// /// /// internal static List ZipGetUtilityFileEntries(string zipFileName) { return ZipGetEntries(GetFullPathForBackupFile(zipFileName)); } /// /// Get zip entries for full path and file name /// returns the entry fullname sorted alphabetically so that folders stay together /// /// /// internal static List ZipGetEntries(string zipPath) { List zipEntries = new List(); using (ZipArchive archive = ZipFile.OpenRead(zipPath)) { foreach (ZipArchiveEntry entry in archive.Entries) { zipEntries.Add(entry.FullName); } } zipEntries.Sort(); return zipEntries; } #endregion Zip handling #region Attachment file handling // /// // /// Get user folder // /// // /// // internal static string AttachmentFilesFolder // { // get // { // return ServerBootConfig.AYANOVA_ATTACHMENT_FILES_PATH; // } // } /// /// Get a random file name with path to attachments folder /// /// internal static string NewRandomAttachmentFilesFolderFileName { get { return Path.Combine(ServerBootConfig.AYANOVA_ATTACHMENT_FILES_PATH, NewRandomFileName); } } /// /// Store a file attachment /// /// internal static async Task StoreFileAttachmentAsync( string tempFilePath, string contentType, string fileName, DateTime lastModified, AyaTypeId attachToObject, string notes, long attachedByUserId, AyContext ct) { //calculate hash var hash = FileHash.GetChecksum(tempFilePath); //Move to folder based on hash var permanentPath = GetPermanentAttachmentPath(hash); EnsurePath(permanentPath); var permanentFilePath = Path.Combine(permanentPath, hash); var FileSize = new FileInfo(tempFilePath).Length; //See if the file was already uploaded, if so then ignore it for now if (File.Exists(permanentFilePath)) { //delete the temp file, it's already stored File.Delete(tempFilePath); } else { System.IO.File.Move(tempFilePath, permanentFilePath); } //seems to be uploaded with the text null if (notes == "null") notes = string.Empty; //Build AyFileInfo FileAttachment fi = new FileAttachment() { StoredFileName = hash, DisplayFileName = fileName, Notes = notes, ContentType = contentType, AttachToObjectId = attachToObject.ObjectId, AttachToAType = attachToObject.AType, LastModified = lastModified, Size = FileSize, AttachedByUserId = attachedByUserId }; //Store in DB await ct.FileAttachment.AddAsync(fi); await ct.SaveChangesAsync(); //Return AyFileInfo object return fi; } /// ///use first three characters for name of folders one character per folder, i.e.: ///if the checksum is f6a5b1236dbba1647257cc4646308326 ///it would be stored in userfiles/f/6/a/f6a5b1236dbba1647257cc4646308326 /// /// /// Path without the file internal static string GetPermanentAttachmentPath(string hash) { return Path.Combine(ServerBootConfig.AYANOVA_ATTACHMENT_FILES_PATH, hash[0].ToString(), hash[1].ToString(), hash[2].ToString()); } /// /// Get the whole path including file name not just the folder /// /// /// internal static string GetPermanentAttachmentFilePath(string hash) { return Path.Combine(ServerBootConfig.AYANOVA_ATTACHMENT_FILES_PATH, hash[0].ToString(), hash[1].ToString(), hash[2].ToString(), hash); } ////////////////////////////////////////////////////////// // // Delete all attachments for object // internal static async Task DeleteAttachmentsForObjectAsync(AyaType ayaType, long ayaId, AyContext ct) { var deleteList = await ct.FileAttachment.Where(z => z.AttachToObjectId == ayaId && z.AttachToAType == ayaType).ToListAsync(); foreach (var d in deleteList) { await DeleteFileAttachmentAsync(d, ct); } } /// /// Delete a file attachment /// checks ref count and if would be zero deletes file physically /// otherwise just deletes pointer in db /// /// /// /// internal static async Task DeleteFileAttachmentAsync(FileAttachment fileAttachmentToBeDeleted, AyContext ct) { //check ref count of file var count = await ct.FileAttachment.LongCountAsync(z => z.StoredFileName == fileAttachmentToBeDeleted.StoredFileName); //Remove from the DB ct.FileAttachment.Remove(fileAttachmentToBeDeleted); await ct.SaveChangesAsync(); if (count < 2) { //remove the file completely var permanentPath = GetPermanentAttachmentPath(fileAttachmentToBeDeleted.StoredFileName); var permanentFilePath = Path.Combine(permanentPath, fileAttachmentToBeDeleted.StoredFileName); if (File.Exists(permanentFilePath)) { //delete the temp file, it's already stored File.Delete(permanentFilePath); } } //Return AyFileInfo object return; } /// /// DANGER: Erases all user files /// internal static void EraseEntireContentsOfAttachmentFilesFolder() { System.IO.DirectoryInfo di = new DirectoryInfo(ServerBootConfig.AYANOVA_ATTACHMENT_FILES_PATH); foreach (FileInfo file in di.EnumerateFiles()) { file.Delete(); } foreach (DirectoryInfo dir in di.EnumerateDirectories()) { dir.Delete(true); } } internal static void BackupAttachments(ILogger log = null) { try { var AttachmentsBackupFile = $"at-{FileUtil.GetSafeDateFileName()}.zip";//presentation issue so don't use UTC for this one AttachmentsBackupFile = GetFullPathForBackupFile(AttachmentsBackupFile); System.IO.Compression.ZipFile.CreateFromDirectory(ServerBootConfig.AYANOVA_ATTACHMENT_FILES_PATH, AttachmentsBackupFile); } catch (Exception ex) { if (log != null) { log.LogError(ex, $"FileUtil::BackupAttachments"); } throw; } } internal static long AttachmentFilesDriveAvailableSpace() { return new System.IO.DriveInfo(Path.GetPathRoot(ServerBootConfig.AYANOVA_ATTACHMENT_FILES_PATH)).AvailableFreeSpace; } internal static IEnumerable GetAllAttachmentFilePaths() { return Directory.EnumerateFiles(ServerBootConfig.AYANOVA_ATTACHMENT_FILES_PATH, "*", SearchOption.AllDirectories); } /// /// Confirm if a file exists in the attachment folder /// /// name of attachment file internal static bool AttachmentFileExists(string fileName) { if (string.IsNullOrWhiteSpace(fileName)) return false; // var utilityFilePath = GetFullPathForBackupFile(fileName); return File.Exists(GetPermanentAttachmentFilePath(fileName)); } internal static bool AppearsToBeAnOrphanedAttachment(string fullPathName) { // is it in the correct folder which is named based on it's hash? // Is it X characters long (they all are or not?) Not sure though actually, maybe they are all 64 characters, maybe not // Does it have an extension? None of ours have an extension if (Path.HasExtension(fullPathName)) return false; var FileName = Path.GetFileName(fullPathName); //2339371F6C0C88656888163072635B282BB7FFF7B33771AB2295C868A0FECD34 //3D67D4D258DCC7BB3CB560013C737E9865DFFB324C2012AA7E9E75CCCBE4133C //BA7816BF8F01CFEA414140DE5DAE2223B00361A396177A9CB410FF61F20015AD //40CE02D157C845E42AA4EF7DCC93A74B0179649C8D0A806B2F985D34AA7385CE //9F2BA2DF87889B1E71346CC575A6F57334B441DB5AE4D40814F95E232C9539B5 //got to be at least 32 chars if (FileName.Length < 32) return false; //probably all 64 chars but let's not count on that and go with folder is correct //what *should* the path be for a file of this name? var ExpectedFullPath = GetPermanentAttachmentFilePath(FileName); //if expected equals real then it's very likely an orphaned file return fullPathName == ExpectedFullPath; } #endregion attachment stuff #region General utilities /// /// Get a random file name, no extension /// /// internal static string NewRandomFileName { get { return Path.GetRandomFileName(); } } //https://stackoverflow.com/a/11124118/8939 // Returns the human-readable file size for an arbitrary, 64-bit file size // The default format is "0.### XB", e.g. "4.2 KB" or "1.434 GB" public static string GetBytesReadable(long i) { // Get absolute value long absolute_i = (i < 0 ? -i : i); // Determine the suffix and readable value string suffix; double readable; if (absolute_i >= 0x1000000000000000) // Exabyte { suffix = "EiB"; readable = (i >> 50); } else if (absolute_i >= 0x4000000000000) // Petabyte { suffix = "PiB"; readable = (i >> 40); } else if (absolute_i >= 0x10000000000) // Terabyte { suffix = "TiB"; readable = (i >> 30); } else if (absolute_i >= 0x40000000) // Gigabyte { suffix = "GiB"; readable = (i >> 20); } else if (absolute_i >= 0x100000) // Megabyte { suffix = "MiB"; readable = (i >> 10); } else if (absolute_i >= 0x400) // Kilobyte { suffix = "KiB"; readable = i; } else { return i.ToString("0 B"); // Byte } // Divide by 1024 to get fractional value readable = (readable / 1024); // Return formatted number with suffix return readable.ToString("0.### ") + suffix; } /// /// Attachments / user files folder size info /// /// internal static FolderSizeInfo GetAttachmentFolderSizeInfo() { return GetDirectorySize(new DirectoryInfo(ServerBootConfig.AYANOVA_ATTACHMENT_FILES_PATH)); } /// /// Utility / backup folder file size info /// /// internal static FolderSizeInfo GetBackupFolderSizeInfo() { return GetDirectorySize(new DirectoryInfo(ServerBootConfig.AYANOVA_BACKUP_FILES_PATH)); } /// /// Calculate disk space usage under . If is provided, /// then return subdirectory disk usages as well, up to levels deep. /// If levels is not provided or is 0, return a list with a single element representing the /// directory specified by . /// /// FROM https://stackoverflow.com/a/28094795/8939 /// /// /// public static FolderSizeInfo GetDirectorySize(DirectoryInfo root, int levels = 0) { var currentDirectory = new FolderSizeInfo(); // Add file sizes. FileInfo[] fis = root.GetFiles(); currentDirectory.Size = 0; foreach (FileInfo fi in fis) { currentDirectory.Size += fi.Length; } // Add subdirectory sizes. DirectoryInfo[] dis = root.GetDirectories(); currentDirectory.Path = root; currentDirectory.SizeWithChildren = currentDirectory.Size; currentDirectory.DirectoryCount = dis.Length; currentDirectory.DirectoryCountWithChildren = dis.Length; currentDirectory.FileCount = fis.Length; currentDirectory.FileCountWithChildren = fis.Length; if (levels >= 0) currentDirectory.Children = new List(); foreach (DirectoryInfo di in dis) { var dd = GetDirectorySize(di, levels - 1); if (levels >= 0) currentDirectory.Children.Add(dd); currentDirectory.SizeWithChildren += dd.SizeWithChildren; currentDirectory.DirectoryCountWithChildren += dd.DirectoryCountWithChildren; currentDirectory.FileCountWithChildren += dd.FileCountWithChildren; } return currentDirectory; } public class FolderSizeInfo { public DirectoryInfo Path { get; set; } public long SizeWithChildren { get; set; } public long Size { get; set; } public long DirectoryCount { get; set; } public long DirectoryCountWithChildren { get; set; } public long FileCount { get; set; } public long FileCountWithChildren { get; set; } public List Children { get; set; } } //Note assume local time because file times as this is used (backup etc) are a presentation issue not a db issue public static string GetSafeDateFileName() { return DateTime.Now.ToString("yyyyMMddHHmmssfff"); } public static string StringToSafeFileName(string fileName) {//https://stackoverflow.com/a/3678296/8939 if (string.IsNullOrWhiteSpace(fileName)) return "no_name"; char[] invalidFileNameChars = Path.GetInvalidFileNameChars(); // Builds a string out of valid chars and an _ for invalid ones var ret = new string(fileName.Select(ch => invalidFileNameChars.Contains(ch) ? '_' : ch).ToArray()); if (string.IsNullOrWhiteSpace(ret)) return "no_name"; return ret; } public static string StringPathDecodeEnvironmentVariables(string path) { if (string.IsNullOrWhiteSpace(path)) { return string.Empty; } //Linux ~ home folder special handling here if (path.Contains('~')) { //return because no environment variables are expended on linux, this is the only special character return path.Replace("~", Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)); } return System.Environment.ExpandEnvironmentVariables(path); } #endregion general utilities #region licensing related utility to qualify upgradability //https://www.meziantou.net/getting-the-date-of-build-of-a-dotnet-assembly-at-runtime.htm public static DateTime GetLinkerTimestampUtc(Assembly assembly) { var location = assembly.Location; return GetLinkerTimestampUtc(location); } public static DateTime GetLinkerTimestampUtc(string filePath) { const int peHeaderOffset = 60; const int linkerTimestampOffset = 8; var bytes = new byte[2048]; using (var file = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { file.Read(bytes, 0, bytes.Length); } var headerPos = BitConverter.ToInt32(bytes, peHeaderOffset); var secondsSince1970 = BitConverter.ToInt32(bytes, headerPos + linkerTimestampOffset); var dt = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); return dt.AddSeconds(secondsSince1970); } #endregion }//eoc }//eons