From 89c411bc286a1f8848d77600746c685b83960372 Mon Sep 17 00:00:00 2001 From: John Cardinal Date: Tue, 16 Mar 2021 17:14:35 +0000 Subject: [PATCH] --- .../ControllerHelpers/ApiUploadProcessor.cs | 141 ++++++++++-------- .../Controllers/AttachmentController.cs | 43 +++--- .../AyaNova/Controllers/ImportController.cs | 4 +- server/AyaNova/Controllers/LogoController.cs | 10 +- server/AyaNova/util/ServerBootConfig.cs | 8 +- 5 files changed, 115 insertions(+), 91 deletions(-) diff --git a/server/AyaNova/ControllerHelpers/ApiUploadProcessor.cs b/server/AyaNova/ControllerHelpers/ApiUploadProcessor.cs index 71dfeeac..7110c9b7 100644 --- a/server/AyaNova/ControllerHelpers/ApiUploadProcessor.cs +++ b/server/AyaNova/ControllerHelpers/ApiUploadProcessor.cs @@ -34,88 +34,97 @@ namespace AyaNova.Api.ControllerHelpers ApiUploadedFilesResult result = new ApiUploadedFilesResult(); FormOptions _defaultFormOptions = new FormOptions(); - - // Used to accumulate all the form url encoded key value pairs in the - // request. - var formAccumulator = new KeyValueAccumulator(); - - var boundary = MultipartRequestHelper.GetBoundary( - MediaTypeHeaderValue.Parse(httpContext.Request.ContentType), - _defaultFormOptions.MultipartBoundaryLengthLimit); - var reader = new MultipartReader(boundary, httpContext.Request.Body); - - var section = await reader.ReadNextSectionAsync(); - - while (section != null) + try { - ContentDispositionHeaderValue contentDisposition; - var hasContentDispositionHeader = ContentDispositionHeaderValue.TryParse(section.ContentDisposition, out contentDisposition); + // Used to accumulate all the form url encoded key value pairs in the + // request. + var formAccumulator = new KeyValueAccumulator(); - if (hasContentDispositionHeader) + var boundary = MultipartRequestHelper.GetBoundary( + MediaTypeHeaderValue.Parse(httpContext.Request.ContentType), + _defaultFormOptions.MultipartBoundaryLengthLimit); + var reader = new MultipartReader(boundary, httpContext.Request.Body); + + var section = await reader.ReadNextSectionAsync(); + + while (section != null) { - if (MultipartRequestHelper.HasFileContentDisposition(contentDisposition)) + ContentDispositionHeaderValue contentDisposition; + var hasContentDispositionHeader = ContentDispositionHeaderValue.TryParse(section.ContentDisposition, out contentDisposition); + + if (hasContentDispositionHeader) { - - string filePathAndName = string.Empty; - var CleanedUploadFileName = contentDisposition.FileName.Value.Replace("\"", ""); - - //get temp file path and temp file name - filePathAndName = FileUtil.NewRandomAttachmentFilesFolderFileName; - - //save to disk - using (var stream = new FileStream(filePathAndName, FileMode.Create)) + if (MultipartRequestHelper.HasFileContentDisposition(contentDisposition)) { - await section.Body.CopyToAsync(stream); - } - result.UploadedFiles.Add(new UploadedFileInfo() - { - InitialUploadedPathName = filePathAndName, - OriginalFileName = CleanedUploadFileName, - MimeType = section.ContentType - }); - } - else if (MultipartRequestHelper.HasFormDataContentDisposition(contentDisposition)) - { - // Content-Disposition: form-data; name="key" - // - // value + string filePathAndName = string.Empty; + var CleanedUploadFileName = contentDisposition.FileName.Value.Replace("\"", ""); - // Do not limit the key name length here because the - // multipart headers length limit is already in effect. - var key = HeaderUtilities.RemoveQuotes(contentDisposition.Name); - var encoding = GetEncoding(section); - using (var streamReader = new StreamReader( - section.Body, - encoding, - detectEncodingFromByteOrderMarks: true, - bufferSize: 1024, - leaveOpen: true)) - { - // The value length limit is enforced by MultipartBodyLengthLimit - var value = await streamReader.ReadToEndAsync(); - if (String.Equals(value, "undefined", StringComparison.OrdinalIgnoreCase)) + //get temp file path and temp file name + filePathAndName = FileUtil.NewRandomAttachmentFilesFolderFileName; + + //save to disk + using (var stream = new FileStream(filePathAndName, FileMode.Create)) { - value = String.Empty; + await section.Body.CopyToAsync(stream); } - formAccumulator.Append(key.Value, value); - - if (formAccumulator.ValueCount > _defaultFormOptions.ValueCountLimit) + result.UploadedFiles.Add(new UploadedFileInfo() { - throw new InvalidDataException($"Form key count limit {_defaultFormOptions.ValueCountLimit} exceeded."); + InitialUploadedPathName = filePathAndName, + OriginalFileName = CleanedUploadFileName, + MimeType = section.ContentType + + }); + } + else if (MultipartRequestHelper.HasFormDataContentDisposition(contentDisposition)) + { + // Content-Disposition: form-data; name="key" + // + // value + + // Do not limit the key name length here because the + // multipart headers length limit is already in effect. + var key = HeaderUtilities.RemoveQuotes(contentDisposition.Name); + var encoding = GetEncoding(section); + using (var streamReader = new StreamReader( + section.Body, + encoding, + detectEncodingFromByteOrderMarks: true, + bufferSize: 1024, + leaveOpen: true)) + { + // The value length limit is enforced by MultipartBodyLengthLimit + var value = await streamReader.ReadToEndAsync(); + if (String.Equals(value, "undefined", StringComparison.OrdinalIgnoreCase)) + { + value = String.Empty; + } + formAccumulator.Append(key.Value, value); + + if (formAccumulator.ValueCount > _defaultFormOptions.ValueCountLimit) + { + throw new InvalidDataException($"Form key count limit {_defaultFormOptions.ValueCountLimit} exceeded."); + } } } } + + // Drains any remaining section body that has not been consumed and + // reads the headers for the next section. + section = await reader.ReadNextSectionAsync(); } - // Drains any remaining section body that has not been consumed and - // reads the headers for the next section. - section = await reader.ReadNextSectionAsync(); + //Get any extra form fields and return them + result.FormFieldData = formAccumulator.GetResults(); + return result; } + catch (Microsoft.AspNetCore.Http.BadHttpRequestException ex) + { + //most commonly here due to file too large + result.Error = $"Code:{ex.StatusCode}, Error: {ex.Message}"; + return result; - //Get any extra form fields and return them - result.FormFieldData = formAccumulator.GetResults(); - return result; + } } @@ -151,11 +160,13 @@ namespace AyaNova.Api.ControllerHelpers { public Dictionary FormFieldData { get; set; } public List UploadedFiles { get; set; } + public string Error { get; set; } public ApiUploadedFilesResult() { FormFieldData = new Dictionary(); UploadedFiles = new List(); + Error = null; } } diff --git a/server/AyaNova/Controllers/AttachmentController.cs b/server/AyaNova/Controllers/AttachmentController.cs index 2d685eb4..eb0dd5df 100644 --- a/server/AyaNova/Controllers/AttachmentController.cs +++ b/server/AyaNova/Controllers/AttachmentController.cs @@ -194,25 +194,18 @@ namespace AyaNova.Api.Controllers - //used to hold extra file data sent by client - // public class UploadFileData - // { - // public string name { get; set; } - // public long lastModified { get; set; } - // } /// /// Upload attachment file - /// + /// Max 10GiB total /// Requires same Authorization roles as object that file is being attached to /// /// /// NameValue list of filenames and attachment id's [Authorize] [HttpPost] - [DisableFormValueModelBinding] - // [RequestSizeLimit(10737418241)]//10737418240 = 10gb https://github.com/aspnet/Announcements/issues/267 - [RequestSizeLimit(1048576)]//1048576 = 1mb for testing + [DisableFormValueModelBinding] + [RequestSizeLimit(ServerBootConfig.MAX_ATTACHMENT_UPLOAD_BYTES)] public async Task UploadAsync() { //Adapted from the example found here: https://docs.microsoft.com/en-us/aspnet/core/mvc/models/file-uploads#uploading-large-files-with-streaming @@ -228,7 +221,7 @@ namespace AyaNova.Api.Controllers if (!MultipartRequestHelper.IsMultipartContentType(Request.ContentType)) return BadRequest(new ApiErrorResponse(ApiErrorCode.INVALID_OPERATION, null, $"Expected a multipart request, but got {Request.ContentType}")); - var uploadFormData = await ApiUploadProcessor.ProcessUploadAsync(HttpContext); + bool badRequest = false; string AttachToObjectType = string.Empty; @@ -237,10 +230,17 @@ namespace AyaNova.Api.Controllers string Notes = string.Empty; List FileData = new List(); - if ( - !uploadFormData.FormFieldData.ContainsKey("FileData") || - !uploadFormData.FormFieldData.ContainsKey("AttachToObjectType") || - !uploadFormData.FormFieldData.ContainsKey("AttachToObjectId")) + var uploadFormData = await ApiUploadProcessor.ProcessUploadAsync(HttpContext); + if (!string.IsNullOrWhiteSpace(uploadFormData.Error)) + { + badRequest = true; + errorMessage = uploadFormData.Error; + } + + if (!badRequest + && (!uploadFormData.FormFieldData.ContainsKey("FileData") + || !uploadFormData.FormFieldData.ContainsKey("AttachToObjectType") + || !uploadFormData.FormFieldData.ContainsKey("AttachToObjectId"))) { badRequest = true; errorMessage = "Missing one or more required FormFieldData values: AttachToObjectType, AttachToObjectId, FileData"; @@ -314,8 +314,17 @@ namespace AyaNova.Api.Controllers { //delete temp files ApiUploadProcessor.DeleteTempUploadFile(uploadFormData); - //return bad request - return BadRequest(new ApiErrorResponse(ApiErrorCode.VALIDATION_INVALID_VALUE, null, errorMessage)); + //file too large is most likely issue so in that case return this localized properly + if (errorMessage.Contains("413")) + { + var TransId = UserTranslationIdFromContext.Id(HttpContext.Items); + return BadRequest(new ApiErrorResponse( + ApiErrorCode.VALIDATION_LENGTH_EXCEEDED, + null, + String.Format(await TranslationBiz.GetTranslationStaticAsync("AyaFileFileTooLarge", TransId, ct), FileUtil.GetBytesReadable(ServerBootConfig.MAX_ATTACHMENT_UPLOAD_BYTES)))); + } + else//not too big, something else + return BadRequest(new ApiErrorResponse(ApiErrorCode.VALIDATION_INVALID_VALUE, null, errorMessage)); } long UserId = UserIdFromContext.Id(HttpContext.Items); diff --git a/server/AyaNova/Controllers/ImportController.cs b/server/AyaNova/Controllers/ImportController.cs index fae741a1..d0f73d65 100644 --- a/server/AyaNova/Controllers/ImportController.cs +++ b/server/AyaNova/Controllers/ImportController.cs @@ -41,13 +41,13 @@ namespace AyaNova.Api.Controllers /// /// Upload and import file - /// Max 100mb total + /// Max 100MiB total /// /// Results [Authorize] [HttpPost("upload")] [DisableFormValueModelBinding] - [RequestSizeLimit(100000000)]//100mb limit https://github.com/aspnet/Announcements/issues/267 + [RequestSizeLimit(AyaNova.Util.ServerBootConfig.MAX_IMPORT_FILE_UPLOAD_BYTES)] public async Task UploadAsync() { //Adapted from the example found here: https://docs.microsoft.com/en-us/aspnet/core/mvc/models/file-uploads#uploading-large-files-with-streaming diff --git a/server/AyaNova/Controllers/LogoController.cs b/server/AyaNova/Controllers/LogoController.cs index 65745bd5..cc7fe216 100644 --- a/server/AyaNova/Controllers/LogoController.cs +++ b/server/AyaNova/Controllers/LogoController.cs @@ -36,7 +36,7 @@ namespace AyaNova.Api.Controllers private readonly AyContext ct; private readonly ILogger log; private readonly ApiServerState serverState; - private const int MAXIMUM_LOGO_SIZE = 512000;//We really don't want it too big or it will slow the fuck down for everything + // private const int MAXIMUM_LOGO_SIZE = 512000;//We really don't want it too big or it will slow the fuck down for everything /// @@ -114,8 +114,8 @@ namespace AyaNova.Api.Controllers [Authorize] [HttpPost("{size}")] //[DisableFormValueModelBinding] - [RequestSizeLimit(MAXIMUM_LOGO_SIZE)]//currently export file is 200kb * 50 maximum at a time = 15mb https://github.com/aspnet/Announcements/issues/267 - public async Task UploadAsync([FromRoute] string size)//, List files /// Logo image file + [RequestSizeLimit(ServerBootConfig.MAX_LOGO_UPLOAD_BYTES)] + public async Task UploadAsync([FromRoute] string size) { //https://docs.microsoft.com/en-us/aspnet/core/mvc/models/file-uploads?view=aspnetcore-3.1#upload-small-files-with-buffered-model-binding-to-a-database @@ -148,8 +148,8 @@ namespace AyaNova.Api.Controllers using (var memoryStream = new MemoryStream()) { await file.CopyToAsync(memoryStream); - if (memoryStream.Length > MAXIMUM_LOGO_SIZE) - return BadRequest(new ApiErrorResponse(ApiErrorCode.VALIDATION_LENGTH_EXCEEDED, null, $"Logo files must be smaller than {MAXIMUM_LOGO_SIZE} maximum")); + if (memoryStream.Length > ServerBootConfig.MAX_LOGO_UPLOAD_BYTES) + return BadRequest(new ApiErrorResponse(ApiErrorCode.VALIDATION_LENGTH_EXCEEDED, null, $"Logo files must be smaller than {ServerBootConfig.MAX_LOGO_UPLOAD_BYTES} maximum")); switch (size) { case "small": diff --git a/server/AyaNova/util/ServerBootConfig.cs b/server/AyaNova/util/ServerBootConfig.cs index 417dab32..aa54c700 100644 --- a/server/AyaNova/util/ServerBootConfig.cs +++ b/server/AyaNova/util/ServerBootConfig.cs @@ -12,10 +12,14 @@ namespace AyaNova.Util internal static class ServerBootConfig { //############################################################################################################ - //STATIC HARD CODED DEFAULTS NOT SET THROUGH CONFIG + //STATIC HARD CODED COMPILE TIME DEFAULTS NOT SET THROUGH CONFIG internal const int FAILED_AUTH_DELAY = 3000;//ms internal const int REPORT_RENDERING_OPERATION_TIMEOUT = 20000;//ms - //############################################################################################################ + //UPLOAD LIMITS + internal const long MAX_ATTACHMENT_UPLOAD_BYTES = 1048576;//10737418241=10gb in bytes, 1048576 = 1mb for testing + internal const long MAX_LOGO_UPLOAD_BYTES = 512000;//Max 500 KiB total (512000 bytes) + internal const long MAX_IMPORT_FILE_UPLOAD_BYTES = 104857600;//100mib limit + //############################################################################################################ //############################ //SEEDING FLAG INTERNAL ONLY