raven/server/AyaNova/util/FileUtil.cs

using System;
using System.Linq;
using System.Threading.Tasks;
using System.IO;
using System.IO.Compression;
using System.Collections.Generic;
using Microsoft.Extensions.Logging;
using Microsoft.EntityFrameworkCore;
using AyaNova.Models;
using AyaNova.Biz;

namespace AyaNova.Util
{
    /* 
    - Quickly generate large files in windows: http://tweaks.com/windows/62755/quickly-generate-large-test-files-in-windows/

     */

    internal static class FileUtil
    {

        #region Folder ensurance
        /// <summary>
        /// Ensurs folders exist and are not identical
        /// Throws an exception of they are found to be identical preventing startup
        /// The reason for this is to prevent a future erase database operation (which erases all attachment files)
        /// from erasing backups which might prevent recovery in case someone accidentally erases their database
        /// </summary>
        /// <param name="contentRootPath"></param>
        /// <returns></returns>
        internal static void EnsureUserAndUtilityFoldersExistAndAreNotIdentical(string contentRootPath)
        {

            //UserFiles
            if (string.IsNullOrWhiteSpace(ServerBootConfig.AYANOVA_FOLDER_USER_FILES))
            {
                ServerBootConfig.AYANOVA_FOLDER_USER_FILES = Path.Combine(contentRootPath, "userfiles");
            }

            //BackupFiles
            if (ServerBootConfig.AYANOVA_FOLDER_BACKUP_FILES == null)
            {
                ServerBootConfig.AYANOVA_FOLDER_BACKUP_FILES = Path.Combine(contentRootPath, "backupfiles");
            }

            //Prevent using the same folder for both
            if (string.Equals(Path.GetFullPath(ServerBootConfig.AYANOVA_FOLDER_USER_FILES), Path.GetFullPath(ServerBootConfig.AYANOVA_FOLDER_BACKUP_FILES), StringComparison.OrdinalIgnoreCase))
            {
                throw new System.NotSupportedException("E1040: The configuration settings AYANOVA_FOLDER_USER_FILES and the AYANOVA_FOLDER_BACKUP_FILES must not point to the exact same location");
            }

            EnsurePath(ServerBootConfig.AYANOVA_FOLDER_USER_FILES);
            EnsurePath(ServerBootConfig.AYANOVA_FOLDER_BACKUP_FILES);
        }

        //create path if doesn't exist already
        private static void EnsurePath(string path)
        {
            if (!Directory.Exists(path))
                Directory.CreateDirectory(path);
        }
        #endregion folder ensurance

        #region Utility file handling


        /// <summary>
        /// Get a path combining supplied file name and backup files folder
        /// </summary>
        /// <returns></returns>
        internal static string GetFullPathForUtilityFile(string fileName)
        {
            return Path.Combine(UtilityFilesFolder, fileName);
        }


        /// <summary>
        /// Get backup folder
        /// </summary>
        /// <returns></returns>
        internal static string UtilityFilesFolder
        {
            get
            {
                return ServerBootConfig.AYANOVA_FOLDER_BACKUP_FILES;
            }
        }


        /// <summary>
        /// Delete a utility file (backup folder file)        
        /// </summary>
        /// <param name="fileName"></param>
        internal static void DeleteUtilityFile(string fileName)
        {
            var utilityFilePath = GetFullPathForUtilityFile(fileName);
            if (File.Exists(utilityFilePath))
            {
                File.Delete(utilityFilePath);
            }
        }


        /// <summary>
        /// Get a list of files in the utility folder
        /// 
        /// </summary>
        /// <param name="searchPattern">search pattern for files desired, leave blank for any </param>
        /// <returns></returns>
        internal static List<string> UtilityFileList(string searchPattern = "*")
        {
            List<string> returnList = new List<string>();
            foreach (string file in Directory.EnumerateFiles(UtilityFilesFolder, searchPattern))
            {
                returnList.Add(Path.GetFileName(file));
            }
            returnList.Sort();
            return returnList;
        }


        /// <summary>
        /// Confirm if a file exists in the utility folder
        /// </summary>
        /// <param name="fileName">name of utility file </param>
        /// <returns>duh!</returns>
        internal static bool UtilityFileExists(string fileName)
        {
            if (string.IsNullOrWhiteSpace(fileName))
                return false;

            var utilityFilePath = GetFullPathForUtilityFile(fileName);
            return File.Exists(utilityFilePath);

        }

        /// <summary>
        /// DANGER: Erases all Utility files including backups etc
        /// </summary>
        internal static void EraseEntireContentsOfUtilityFilesFolder()
        {
            System.IO.DirectoryInfo di = new DirectoryInfo(UtilityFilesFolder);
            foreach (FileInfo file in di.EnumerateFiles())
            {
                file.Delete();
            }
            foreach (DirectoryInfo dir in di.EnumerateDirectories())
            {
                dir.Delete(true);
            }
        }



        /// <summary>
        /// Cleanup excess backups (backup folder file)        
        /// </summary>
        /// <param name="keepCount"></param>
        internal static void DatabaseBackupCleanUp(int keepCount)
        {
            if (keepCount < 1) keepCount = 1;
            var BackupFileList = UtilityFileList("db-*.backup");

            if (BackupFileList.Count > keepCount)
            {
                //sort, skip newest x (keepcount) delete the rest
                var DeleteCount = BackupFileList.Count - keepCount;
                var DeleteFileList = BackupFileList.OrderByDescending(m => m).Skip(keepCount).ToList();
                foreach (string ExtraBackupFile in DeleteFileList)
                {
                    DeleteUtilityFile(ExtraBackupFile);
                }
            }
        }

        /// <summary>
        /// Cleanup excess backups (backup folder file)        
        /// </summary>
        /// <param name="keepCount"></param>
        internal static void AttachmentBackupCleanUp(int keepCount)
        {
            if (keepCount < 1) keepCount = 1;
            var BackupFileList = UtilityFileList("at-*.zip");

            if (BackupFileList.Count > keepCount)
            {
                //sort, skip newest x (keepcount) delete the rest
                var DeleteCount = BackupFileList.Count - keepCount;
                var DeleteFileList = BackupFileList.OrderByDescending(m => m).Skip(keepCount).ToList();
                foreach (string ExtraBackupFile in DeleteFileList)
                {
                    DeleteUtilityFile(ExtraBackupFile);
                }
            }
        }

        #endregion Utility file handling

        #region Zip handling
        ////////////////////////////////////////////////////////////////////////////////////////
        //ZIP handling

        /// <summary>
        /// Get zip entries for a utlity file
        /// </summary>
        /// <param name="zipFileName"></param>
        /// <returns></returns>
        internal static List<string> ZipGetUtilityFileEntries(string zipFileName)
        {
            return ZipGetEntries(GetFullPathForUtilityFile(zipFileName));
        }

        /// <summary>
        /// Get zip entries for full path and file name
        /// returns the entry fullname sorted alphabetically so that folders stay together
        /// </summary>
        /// <param name="zipPath"></param>
        /// <returns></returns>
        internal static List<string> ZipGetEntries(string zipPath)
        {
            List<string> zipEntries = new List<string>();
            using (ZipArchive archive = ZipFile.OpenRead(zipPath))
            {
                foreach (ZipArchiveEntry entry in archive.Entries)
                {
                    zipEntries.Add(entry.FullName);
                }
            }
            zipEntries.Sort();
            return zipEntries;
        }

        // /// <summary>
        // /// Import utility - get individual files specified in zip archive as JSON objects
        // /// 
        // /// </summary>
        // /// <param name="zipFileName">Name of utility zip import file</param>
        // /// <param name="entryList">Name of entries in utility file archive to fetch</param>
        // /// <returns></returns>
        // internal static List<JObject> ZipGetUtilityArchiveEntriesAsJsonObjects(List<string> entryList, string zipFileName)
        // {
        //     List<JObject> jList = new List<JObject>();
        //     var zipPath = GetFullPathForUtilityFile(zipFileName);
        //     using (ZipArchive archive = ZipFile.OpenRead(zipPath))
        //     {
        //         foreach (string importFileName in entryList)
        //         {
        //             ZipArchiveEntry entry = archive.GetEntry(importFileName);
        //             if (entry != null)
        //             {
        //                 //stream entry into a new jobject and add it to the list
        //                 StreamReader reader = new StreamReader(entry.Open());
        //                 string text = reader.ReadToEnd();
        //                 var j = JObject.Parse(text);

        //                 //Here add v7 import file name as sometimes it's needed later (Translations)
        //                 j.Add("V7_SOURCE_FILE_NAME", JToken.FromObject(importFileName));
        //                 jList.Add(j);
        //             }
        //         }

        //     }

        //     return jList;

        // }


        #endregion Zip handling

        #region Attachment file handling

        /// <summary>
        /// Get user folder
        /// </summary>
        /// <returns></returns>
        internal static string UserFilesFolder
        {
            get
            {
                return ServerBootConfig.AYANOVA_FOLDER_USER_FILES;
            }
        }


        /// <summary>
        /// Get a random file name
        /// </summary>
        /// <returns></returns>
        internal static string NewRandomFileName
        {
            get
            {
                return Path.GetRandomFileName();
            }
        }

        /// <summary>
        /// Get a random file name with path to attachments folder
        /// </summary>
        /// <returns></returns>
        internal static string NewRandomAttachmentFileName
        {
            get
            {
                return Path.Combine(UserFilesFolder, NewRandomFileName);
            }
        }



        /// <summary>
        /// Store a file attachment
        /// </summary>      
        /// <returns></returns>
        internal static async Task<FileAttachment> StoreFileAttachmentAsync(string tempFilePath, string contentType, string fileName, DateTime lastModified,
         AyaTypeId attachToObject, string notes, AyContext ct)
        {
            //calculate hash
            var hash = FileHash.GetChecksum(tempFilePath);

            //Move to folder based on hash
            var permanentPath = GetPermanentAttachmentPath(hash);
            EnsurePath(permanentPath);

            var permanentFilePath = Path.Combine(permanentPath, hash);

            //See if the file was already uploaded, if so then ignore it for now
            if (File.Exists(permanentFilePath))
            {
                //delete the temp file, it's already stored
                File.Delete(tempFilePath);
            }
            else
            {
                System.IO.File.Move(tempFilePath, permanentFilePath);
            }

            //seems to be uploaded with the text null
            if (notes == "null") notes = string.Empty;

            //Build AyFileInfo
            FileAttachment fi = new FileAttachment()
            {
                StoredFileName = hash,
                DisplayFileName = fileName,
                Notes = notes,
                ContentType = contentType,
                AttachToObjectId = attachToObject.ObjectId,
                AttachToObjectType = attachToObject.ObjectType,
                LastModified = lastModified

            };

            //Store in DB
            await ct.FileAttachment.AddAsync(fi);
            await ct.SaveChangesAsync();

            //Return AyFileInfo object
            return fi;

        }

        /// <summary>
        ///use first three characters for name of folders one character per folder, i.e.:
        ///if the checksum is f6a5b1236dbba1647257cc4646308326
        ///it would be stored in userfiles/f/6/a/f6a5b1236dbba1647257cc4646308326
        /// </summary>
        /// <param name="hash"></param>
        /// <returns>Path without the file</returns>
        internal static string GetPermanentAttachmentPath(string hash)
        {
            return Path.Combine(UserFilesFolder, hash[0].ToString(), hash[1].ToString(), hash[2].ToString());
        }

        /// <summary>
        /// Get the whole path including file name not just the folder
        /// </summary>
        /// <param name="hash"></param>
        /// <returns></returns>
        internal static string GetPermanentAttachmentFilePath(string hash)
        {
            return Path.Combine(UserFilesFolder, hash[0].ToString(), hash[1].ToString(), hash[2].ToString(), hash);
        }


        /// <summary>
        /// Delete a file attachment
        /// checks ref count and if would be zero deletes file physically
        /// otherwise just deletes pointer in db
        /// </summary>
        /// <param name="fileAttachmentToBeDeleted"></param>
        /// <param name="ct"></param>
        /// <returns></returns>
        internal static async Task<FileAttachment> DeleteFileAttachmentAsync(FileAttachment fileAttachmentToBeDeleted, AyContext ct)
        {

            //check ref count of file
            var count = await ct.FileAttachment.LongCountAsync(z => z.StoredFileName == fileAttachmentToBeDeleted.StoredFileName);

            //Store in DB
            ct.FileAttachment.Remove(fileAttachmentToBeDeleted);
            await ct.SaveChangesAsync();

            if (count < 2)
            {
                //remove the file completely
                var permanentPath = GetPermanentAttachmentPath(fileAttachmentToBeDeleted.StoredFileName);
                var permanentFilePath = Path.Combine(permanentPath, fileAttachmentToBeDeleted.StoredFileName);

                if (File.Exists(permanentFilePath))
                {
                    //delete the temp file, it's already stored
                    File.Delete(permanentFilePath);
                }
            }

            //Return AyFileInfo object
            return fileAttachmentToBeDeleted;

        }

        /// <summary>
        /// DANGER: Erases all user files
        /// </summary>
        internal static void EraseEntireContentsOfUserFilesFolder()
        {
            System.IO.DirectoryInfo di = new DirectoryInfo(UserFilesFolder);
            foreach (FileInfo file in di.EnumerateFiles())
            {
                file.Delete();
            }
            foreach (DirectoryInfo dir in di.EnumerateDirectories())
            {
                dir.Delete(true);
            }
        }


        internal static void BackupAttachments(ILogger log = null)
        {
            try
            {
                var AttachmentsBackupFile = $"at-{FileUtil.GetSafeDateFileName()}.zip";//presentation issue so don't use UTC for this one
                AttachmentsBackupFile = GetFullPathForUtilityFile(AttachmentsBackupFile);

                System.IO.Compression.ZipFile.CreateFromDirectory(UserFilesFolder, AttachmentsBackupFile);
            }
            catch (Exception ex)
            {
                if (log != null)
                {
                    log.LogError(ex, $"FileUtil::BackupAttachments");
                }
                throw ex;
            }
        }


        #endregion attachment stuff

        #region General utilities

        /// <summary>
        /// Attachments / user files folder size info
        /// </summary>
        /// <returns></returns>
        internal static FolderSizeInfo GetAttachmentFolderSizeInfo()
        {
            return GetDirectorySize(new DirectoryInfo(UserFilesFolder));
        }

        /// <summary>
        /// Utility / backup folder file size info
        /// </summary>
        /// <returns></returns>
        internal static FolderSizeInfo GetUtilityFolderSizeInfo()
        {
            return GetDirectorySize(new DirectoryInfo(UtilityFilesFolder));
        }


        /// <summary>
        /// Calculate disk space usage under <paramref name="root"/>.  If <paramref name="levels"/> is provided, 
        /// then return subdirectory disk usages as well, up to <paramref name="levels"/> levels deep.
        /// If levels is not provided or is 0, return a list with a single element representing the
        /// directory specified by <paramref name="root"/>.
        /// 
        /// FROM https://stackoverflow.com/a/28094795/8939
        /// 
        /// </summary>
        /// <returns></returns>
        public static FolderSizeInfo GetDirectorySize(DirectoryInfo root, int levels = 0)
        {
            var currentDirectory = new FolderSizeInfo();

            // Add file sizes.
            FileInfo[] fis = root.GetFiles();
            currentDirectory.Size = 0;
            foreach (FileInfo fi in fis)
            {
                currentDirectory.Size += fi.Length;
            }

            // Add subdirectory sizes.
            DirectoryInfo[] dis = root.GetDirectories();

            currentDirectory.Path = root;
            currentDirectory.SizeWithChildren = currentDirectory.Size;
            currentDirectory.DirectoryCount = dis.Length;
            currentDirectory.DirectoryCountWithChildren = dis.Length;
            currentDirectory.FileCount = fis.Length;
            currentDirectory.FileCountWithChildren = fis.Length;

            if (levels >= 0)
                currentDirectory.Children = new List<FolderSizeInfo>();

            foreach (DirectoryInfo di in dis)
            {
                var dd = GetDirectorySize(di, levels - 1);
                if (levels >= 0)
                    currentDirectory.Children.Add(dd);

                currentDirectory.SizeWithChildren += dd.SizeWithChildren;
                currentDirectory.DirectoryCountWithChildren += dd.DirectoryCountWithChildren;
                currentDirectory.FileCountWithChildren += dd.FileCountWithChildren;
            }

            return currentDirectory;
        }

        public class FolderSizeInfo
        {
            public DirectoryInfo Path { get; set; }
            public long SizeWithChildren { get; set; }
            public long Size { get; set; }
            public int DirectoryCount { get; set; }
            public int DirectoryCountWithChildren { get; set; }
            public int FileCount { get; set; }
            public int FileCountWithChildren { get; set; }
            public List<FolderSizeInfo> Children { get; set; }
        }


        //Note assume local time because file times as this is used (backup etc) are a presentation issue not a db issue
        public static string GetSafeDateFileName()
        {
            return DateTime.Now.ToString("yyyyMMddHHmmssfff");
        }

        #endregion general utilities

    }//eoc

}//eons