Files
raven/server/AyaNova/util/FileUtil.cs
2020-06-25 16:00:39 +00:00

664 lines
23 KiB
C#

using System;
using System.Linq;
using System.Threading.Tasks;
using System.IO;
using System.IO.Compression;
using System.Collections.Generic;
using Microsoft.Extensions.Logging;
using Microsoft.EntityFrameworkCore;
using AyaNova.Models;
using AyaNova.Biz;
namespace AyaNova.Util
{
/*
- Quickly generate large files in windows: http://tweaks.com/windows/62755/quickly-generate-large-test-files-in-windows/
*/
internal static class FileUtil
{
#region Folder ensurance
/// <summary>
/// Ensurs folders exist and are not identical
/// Throws an exception of they are found to be identical preventing startup
/// The reason for this is to prevent a future erase database operation (which erases all attachment files)
/// from erasing backups which might prevent recovery in case someone accidentally erases their database
/// </summary>
/// <param name="contentRootPath"></param>
/// <returns></returns>
internal static void EnsureUserAndUtilityFoldersExistAndAreNotIdentical(string contentRootPath)
{
//UserFiles
if (string.IsNullOrWhiteSpace(ServerBootConfig.AYANOVA_FOLDER_USER_FILES))
{
ServerBootConfig.AYANOVA_FOLDER_USER_FILES = Path.Combine(contentRootPath, "userfiles");
}
//BackupFiles
if (ServerBootConfig.AYANOVA_FOLDER_BACKUP_FILES == null)
{
ServerBootConfig.AYANOVA_FOLDER_BACKUP_FILES = Path.Combine(contentRootPath, "backupfiles");
}
//Prevent using the same folder for both
if (string.Equals(Path.GetFullPath(ServerBootConfig.AYANOVA_FOLDER_USER_FILES), Path.GetFullPath(ServerBootConfig.AYANOVA_FOLDER_BACKUP_FILES), StringComparison.OrdinalIgnoreCase))
{
throw new System.NotSupportedException("E1040: The configuration settings AYANOVA_FOLDER_USER_FILES and the AYANOVA_FOLDER_BACKUP_FILES must not point to the exact same location");
}
EnsurePath(ServerBootConfig.AYANOVA_FOLDER_USER_FILES);
EnsurePath(ServerBootConfig.AYANOVA_FOLDER_BACKUP_FILES);
}
//create path if doesn't exist already
private static void EnsurePath(string path)
{
if (!Directory.Exists(path))
Directory.CreateDirectory(path);
}
#endregion folder ensurance
#region Utility file handling
/// <summary>
/// Get a path combining supplied file name and backup files folder
/// </summary>
/// <returns></returns>
internal static string GetFullPathForUtilityFile(string fileName)
{
return Path.Combine(UtilityFilesFolder, fileName);
}
/// <summary>
/// Get backup folder
/// </summary>
/// <returns></returns>
internal static string UtilityFilesFolder
{
get
{
return ServerBootConfig.AYANOVA_FOLDER_BACKUP_FILES;
}
}
public class BackupFileInfo
{
public string length { get; set; }
public string Name { get; set; }
public DateTime Created { get; set; }
}
public class BackupStatus
{
public string AvailableFreeSpace { get; set; }
public List<BackupFileInfo> BackupFiles { get; set; }
public BackupStatus()
{
AvailableFreeSpace = null;
BackupFiles = new List<BackupFileInfo>();
}
}
/// <summary>
/// Get a status report of backup
/// for reporting to ops user in UI
/// </summary>
/// <returns></returns>
internal static BackupStatus BackupStatusReport()
{
BackupStatus statusReport = new BackupStatus();
try
{
statusReport.AvailableFreeSpace = GetBytesReadable(new System.IO.DriveInfo(Path.GetPathRoot(UtilityFilesFolder)).AvailableFreeSpace);
}
catch (Exception ex)
{
statusReport.AvailableFreeSpace = "ERROR";
ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("FileUtil::BackupStatus");
log.LogError(ex, "FileUtil::BackupStatusReport error getting available space");
}
var backupFiles = Directory.EnumerateFiles(UtilityFilesFolder, "*");
foreach (string file in backupFiles.OrderByDescending(z => z))
{
var fi = new FileInfo(file);
statusReport.BackupFiles.Add(new BackupFileInfo()
{
Name = Path.GetFileName(file),
length = GetBytesReadable(fi.Length),
Created = fi.CreationTimeUtc
});
}
return statusReport;
}
// List<string> returnList = new List<string>();
// foreach (string file in Directory.EnumerateFiles(UtilityFilesFolder, "*"))
// {
// var fi = new FileInfo(file);
// returnList.Add(fi.Length);
// }
// returnList.Sort();
// return returnList;
/// <summary>
/// Get date of newest automatic backup file or minvalue if not found
///
/// </summary>
/// <returns></returns>
internal static DateTime MostRecentAutomatedBackupFileDate()
{
DateTime LastBackup = DateTime.MinValue;
var BackupPath = UtilityFilesFolder;
foreach (string file in Directory.EnumerateFiles(UtilityFilesFolder, "db-*.backup"))
{
var ThisFileTime = File.GetCreationTimeUtc(file);
if (ThisFileTime > LastBackup)
{
LastBackup = ThisFileTime;
}
}
return LastBackup;
}
/// <summary>
/// Confirm if a file exists in the utility folder
/// </summary>
/// <param name="fileName">name of utility file </param>
/// <returns>duh!</returns>
internal static bool UtilityFileExists(string fileName)
{
if (string.IsNullOrWhiteSpace(fileName))
return false;
var utilityFilePath = GetFullPathForUtilityFile(fileName);
return File.Exists(utilityFilePath);
}
/// <summary>
/// DANGER: Erases all Utility files including backups etc
/// </summary>
internal static void EraseEntireContentsOfUtilityFilesFolder()
{
System.IO.DirectoryInfo di = new DirectoryInfo(UtilityFilesFolder);
foreach (FileInfo file in di.EnumerateFiles())
{
file.Delete();
}
foreach (DirectoryInfo dir in di.EnumerateDirectories())
{
dir.Delete(true);
}
}
/// <summary>
/// Cleanup excess backups (backup folder file)
/// </summary>
/// <param name="keepCount"></param>
internal static void DatabaseBackupCleanUp(int keepCount)
{
if (keepCount < 1) keepCount = 1;
//Database backups
var BackupFileList = Directory.EnumerateFiles(UtilityFilesFolder, "db-*.backup");
if (BackupFileList.Count() > keepCount)
{
//sort, skip newest x (keepcount) delete the rest
var DeleteCount = BackupFileList.Count() - keepCount;
var DeleteFileList = BackupFileList.OrderByDescending(m => m).Skip(keepCount).ToList();
foreach (string ExtraBackupFile in DeleteFileList)
{
File.Delete(ExtraBackupFile);
}
}
//Attachment backups
BackupFileList = Directory.EnumerateFiles(UtilityFilesFolder, "at-*.zip");
if (BackupFileList.Count() > keepCount)
{
//sort, skip newest x (keepcount) delete the rest
var DeleteCount = BackupFileList.Count() - keepCount;
var DeleteFileList = BackupFileList.OrderByDescending(m => m).Skip(keepCount).ToList();
foreach (string ExtraBackupFile in DeleteFileList)
{
File.Delete(ExtraBackupFile);
}
}
}
internal static long UtilityFilesDriveAvailableSpace()
{
try
{
return new System.IO.DriveInfo(Path.GetPathRoot(UtilityFilesFolder)).AvailableFreeSpace;
}
catch (Exception ex)
{
ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("FileUtil::UtilityFilesDriveAvailableSpace");
log.LogError(ex, "FileUtil::UtilityFilesDriveAvailableSpace error getting available space");
return 0;
}
}
#endregion Utility file handling
#region Zip handling
////////////////////////////////////////////////////////////////////////////////////////
//ZIP handling
/// <summary>
/// Get zip entries for a utlity file
/// </summary>
/// <param name="zipFileName"></param>
/// <returns></returns>
internal static List<string> ZipGetUtilityFileEntries(string zipFileName)
{
return ZipGetEntries(GetFullPathForUtilityFile(zipFileName));
}
/// <summary>
/// Get zip entries for full path and file name
/// returns the entry fullname sorted alphabetically so that folders stay together
/// </summary>
/// <param name="zipPath"></param>
/// <returns></returns>
internal static List<string> ZipGetEntries(string zipPath)
{
List<string> zipEntries = new List<string>();
using (ZipArchive archive = ZipFile.OpenRead(zipPath))
{
foreach (ZipArchiveEntry entry in archive.Entries)
{
zipEntries.Add(entry.FullName);
}
}
zipEntries.Sort();
return zipEntries;
}
#endregion Zip handling
#region Attachment file handling
/// <summary>
/// Get user folder
/// </summary>
/// <returns></returns>
internal static string UserFilesFolder
{
get
{
return ServerBootConfig.AYANOVA_FOLDER_USER_FILES;
}
}
/// <summary>
/// Get a random file name
/// </summary>
/// <returns></returns>
internal static string NewRandomFileName
{
get
{
return Path.GetRandomFileName();
}
}
/// <summary>
/// Get a random file name with path to attachments folder
/// </summary>
/// <returns></returns>
internal static string NewRandomAttachmentFileName
{
get
{
return Path.Combine(UserFilesFolder, NewRandomFileName);
}
}
/// <summary>
/// Store a file attachment
/// </summary>
/// <returns></returns>
internal static async Task<FileAttachment> StoreFileAttachmentAsync(string tempFilePath, string contentType, string fileName, DateTime lastModified,
AyaTypeId attachToObject, string notes, AyContext ct)
{
//calculate hash
var hash = FileHash.GetChecksum(tempFilePath);
//Move to folder based on hash
var permanentPath = GetPermanentAttachmentPath(hash);
EnsurePath(permanentPath);
var permanentFilePath = Path.Combine(permanentPath, hash);
//See if the file was already uploaded, if so then ignore it for now
if (File.Exists(permanentFilePath))
{
//delete the temp file, it's already stored
File.Delete(tempFilePath);
}
else
{
System.IO.File.Move(tempFilePath, permanentFilePath);
}
//seems to be uploaded with the text null
if (notes == "null") notes = string.Empty;
//Build AyFileInfo
FileAttachment fi = new FileAttachment()
{
StoredFileName = hash,
DisplayFileName = fileName,
Notes = notes,
ContentType = contentType,
AttachToObjectId = attachToObject.ObjectId,
AttachToObjectType = attachToObject.ObjectType,
LastModified = lastModified
};
//Store in DB
await ct.FileAttachment.AddAsync(fi);
await ct.SaveChangesAsync();
//Return AyFileInfo object
return fi;
}
/// <summary>
///use first three characters for name of folders one character per folder, i.e.:
///if the checksum is f6a5b1236dbba1647257cc4646308326
///it would be stored in userfiles/f/6/a/f6a5b1236dbba1647257cc4646308326
/// </summary>
/// <param name="hash"></param>
/// <returns>Path without the file</returns>
internal static string GetPermanentAttachmentPath(string hash)
{
return Path.Combine(UserFilesFolder, hash[0].ToString(), hash[1].ToString(), hash[2].ToString());
}
/// <summary>
/// Get the whole path including file name not just the folder
/// </summary>
/// <param name="hash"></param>
/// <returns></returns>
internal static string GetPermanentAttachmentFilePath(string hash)
{
return Path.Combine(UserFilesFolder, hash[0].ToString(), hash[1].ToString(), hash[2].ToString(), hash);
}
/// <summary>
/// Delete a file attachment
/// checks ref count and if would be zero deletes file physically
/// otherwise just deletes pointer in db
/// </summary>
/// <param name="fileAttachmentToBeDeleted"></param>
/// <param name="ct"></param>
/// <returns></returns>
internal static async Task<FileAttachment> DeleteFileAttachmentAsync(FileAttachment fileAttachmentToBeDeleted, AyContext ct)
{
//check ref count of file
var count = await ct.FileAttachment.LongCountAsync(z => z.StoredFileName == fileAttachmentToBeDeleted.StoredFileName);
//Store in DB
ct.FileAttachment.Remove(fileAttachmentToBeDeleted);
await ct.SaveChangesAsync();
if (count < 2)
{
//remove the file completely
var permanentPath = GetPermanentAttachmentPath(fileAttachmentToBeDeleted.StoredFileName);
var permanentFilePath = Path.Combine(permanentPath, fileAttachmentToBeDeleted.StoredFileName);
if (File.Exists(permanentFilePath))
{
//delete the temp file, it's already stored
File.Delete(permanentFilePath);
}
}
//Return AyFileInfo object
return fileAttachmentToBeDeleted;
}
/// <summary>
/// DANGER: Erases all user files
/// </summary>
internal static void EraseEntireContentsOfUserFilesFolder()
{
System.IO.DirectoryInfo di = new DirectoryInfo(UserFilesFolder);
foreach (FileInfo file in di.EnumerateFiles())
{
file.Delete();
}
foreach (DirectoryInfo dir in di.EnumerateDirectories())
{
dir.Delete(true);
}
}
internal static void BackupAttachments(string demandFileNamePrepend, ILogger log = null)
{
try
{
var AttachmentsBackupFile = $"{demandFileNamePrepend}at-{FileUtil.GetSafeDateFileName()}.zip";//presentation issue so don't use UTC for this one
AttachmentsBackupFile = GetFullPathForUtilityFile(AttachmentsBackupFile);
System.IO.Compression.ZipFile.CreateFromDirectory(UserFilesFolder, AttachmentsBackupFile);
}
catch (Exception ex)
{
if (log != null)
{
log.LogError(ex, $"FileUtil::BackupAttachments");
}
throw ex;
}
}
internal static long AttachmentFilesDriveAvailableSpace()
{
try
{
return new System.IO.DriveInfo(Path.GetPathRoot(UserFilesFolder)).AvailableFreeSpace;
}
catch (Exception ex)
{
ILogger log = AyaNova.Util.ApplicationLogging.CreateLogger("FileUtil::AttachmentFilesDriveAvailableSpace");
log.LogError(ex, "FileUtil::AttachmentFilesDriveAvailableSpace error getting available space");
return 0;
}
}
#endregion attachment stuff
#region General utilities
//https://stackoverflow.com/a/11124118/8939
// Returns the human-readable file size for an arbitrary, 64-bit file size
// The default format is "0.### XB", e.g. "4.2 KB" or "1.434 GB"
public static string GetBytesReadable(long i)
{
// Get absolute value
long absolute_i = (i < 0 ? -i : i);
// Determine the suffix and readable value
string suffix;
double readable;
if (absolute_i >= 0x1000000000000000) // Exabyte
{
suffix = "EB";
readable = (i >> 50);
}
else if (absolute_i >= 0x4000000000000) // Petabyte
{
suffix = "PB";
readable = (i >> 40);
}
else if (absolute_i >= 0x10000000000) // Terabyte
{
suffix = "TB";
readable = (i >> 30);
}
else if (absolute_i >= 0x40000000) // Gigabyte
{
suffix = "GB";
readable = (i >> 20);
}
else if (absolute_i >= 0x100000) // Megabyte
{
suffix = "MB";
readable = (i >> 10);
}
else if (absolute_i >= 0x400) // Kilobyte
{
suffix = "KB";
readable = i;
}
else
{
return i.ToString("0 B"); // Byte
}
// Divide by 1024 to get fractional value
readable = (readable / 1024);
// Return formatted number with suffix
return readable.ToString("0.### ") + suffix;
}
/// <summary>
/// Attachments / user files folder size info
/// </summary>
/// <returns></returns>
internal static FolderSizeInfo GetAttachmentFolderSizeInfo()
{
return GetDirectorySize(new DirectoryInfo(UserFilesFolder));
}
/// <summary>
/// Utility / backup folder file size info
/// </summary>
/// <returns></returns>
internal static FolderSizeInfo GetUtilityFolderSizeInfo()
{
return GetDirectorySize(new DirectoryInfo(UtilityFilesFolder));
}
/// <summary>
/// Calculate disk space usage under <paramref name="root"/>. If <paramref name="levels"/> is provided,
/// then return subdirectory disk usages as well, up to <paramref name="levels"/> levels deep.
/// If levels is not provided or is 0, return a list with a single element representing the
/// directory specified by <paramref name="root"/>.
///
/// FROM https://stackoverflow.com/a/28094795/8939
///
/// </summary>
/// <returns></returns>
public static FolderSizeInfo GetDirectorySize(DirectoryInfo root, int levels = 0)
{
var currentDirectory = new FolderSizeInfo();
// Add file sizes.
FileInfo[] fis = root.GetFiles();
currentDirectory.Size = 0;
foreach (FileInfo fi in fis)
{
currentDirectory.Size += fi.Length;
}
// Add subdirectory sizes.
DirectoryInfo[] dis = root.GetDirectories();
currentDirectory.Path = root;
currentDirectory.SizeWithChildren = currentDirectory.Size;
currentDirectory.DirectoryCount = dis.Length;
currentDirectory.DirectoryCountWithChildren = dis.Length;
currentDirectory.FileCount = fis.Length;
currentDirectory.FileCountWithChildren = fis.Length;
if (levels >= 0)
currentDirectory.Children = new List<FolderSizeInfo>();
foreach (DirectoryInfo di in dis)
{
var dd = GetDirectorySize(di, levels - 1);
if (levels >= 0)
currentDirectory.Children.Add(dd);
currentDirectory.SizeWithChildren += dd.SizeWithChildren;
currentDirectory.DirectoryCountWithChildren += dd.DirectoryCountWithChildren;
currentDirectory.FileCountWithChildren += dd.FileCountWithChildren;
}
return currentDirectory;
}
public class FolderSizeInfo
{
public DirectoryInfo Path { get; set; }
public long SizeWithChildren { get; set; }
public long Size { get; set; }
public long DirectoryCount { get; set; }
public long DirectoryCountWithChildren { get; set; }
public long FileCount { get; set; }
public long FileCountWithChildren { get; set; }
public List<FolderSizeInfo> Children { get; set; }
}
//Note assume local time because file times as this is used (backup etc) are a presentation issue not a db issue
public static string GetSafeDateFileName()
{
return DateTime.Now.ToString("yyyyMMddHHmmssfff");
}
public static string StringToSafeFileName(string fileName)
{//https://stackoverflow.com/a/3678296/8939
if (string.IsNullOrWhiteSpace(fileName))
return "no_name";
char[] invalidFileNameChars = Path.GetInvalidFileNameChars();
// Builds a string out of valid chars and an _ for invalid ones
var ret = new string(fileName.Select(ch => invalidFileNameChars.Contains(ch) ? '_' : ch).ToArray());
if (string.IsNullOrWhiteSpace(ret))
return "no_name";
return ret;
}
#endregion general utilities
}//eoc
}//eons