admin/raven
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Browse Source
This commit is contained in:
John Cardinal
2021-03-16 17:14:35 +00:00
parent 9236c91ff6
commit 89c411bc28
5 changed files with 115 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

141
server/AyaNova/ControllerHelpers/ApiUploadProcessor.cs
View File

@@ -34,88 +34,97 @@ namespace AyaNova.Api.ControllerHelpers
ApiUploadedFilesResult result = new ApiUploadedFilesResult(); ApiUploadedFilesResult result = new ApiUploadedFilesResult();
FormOptions _defaultFormOptions = new FormOptions(); FormOptions _defaultFormOptions = new FormOptions();
try
// Used to accumulate all the form url encoded key value pairs in the
// request.
var formAccumulator = new KeyValueAccumulator();
var boundary = MultipartRequestHelper.GetBoundary(
MediaTypeHeaderValue.Parse(httpContext.Request.ContentType),
_defaultFormOptions.MultipartBoundaryLengthLimit);
var reader = new MultipartReader(boundary, httpContext.Request.Body);
var section = await reader.ReadNextSectionAsync();
while (section != null)
{ {
ContentDispositionHeaderValue contentDisposition; // Used to accumulate all the form url encoded key value pairs in the
var hasContentDispositionHeader = ContentDispositionHeaderValue.TryParse(section.ContentDisposition, out contentDisposition); // request.
var formAccumulator = new KeyValueAccumulator();
if (hasContentDispositionHeader) var boundary = MultipartRequestHelper.GetBoundary(
MediaTypeHeaderValue.Parse(httpContext.Request.ContentType),
_defaultFormOptions.MultipartBoundaryLengthLimit);
var reader = new MultipartReader(boundary, httpContext.Request.Body);
var section = await reader.ReadNextSectionAsync();
while (section != null)
{ {
if (MultipartRequestHelper.HasFileContentDisposition(contentDisposition)) ContentDispositionHeaderValue contentDisposition;
var hasContentDispositionHeader = ContentDispositionHeaderValue.TryParse(section.ContentDisposition, out contentDisposition);
if (hasContentDispositionHeader)
{ {
if (MultipartRequestHelper.HasFileContentDisposition(contentDisposition))
string filePathAndName = string.Empty;
var CleanedUploadFileName = contentDisposition.FileName.Value.Replace("\"", "");
//get temp file path and temp file name
filePathAndName = FileUtil.NewRandomAttachmentFilesFolderFileName;
//save to disk
using (var stream = new FileStream(filePathAndName, FileMode.Create))
{ {
await section.Body.CopyToAsync(stream);
}
result.UploadedFiles.Add(new UploadedFileInfo()
{
InitialUploadedPathName = filePathAndName,
OriginalFileName = CleanedUploadFileName,
MimeType = section.ContentType
}); string filePathAndName = string.Empty;
} var CleanedUploadFileName = contentDisposition.FileName.Value.Replace("\"", "");
else if (MultipartRequestHelper.HasFormDataContentDisposition(contentDisposition))
{
// Content-Disposition: form-data; name="key"
//
// value
// Do not limit the key name length here because the //get temp file path and temp file name
// multipart headers length limit is already in effect. filePathAndName = FileUtil.NewRandomAttachmentFilesFolderFileName;
var key = HeaderUtilities.RemoveQuotes(contentDisposition.Name);
var encoding = GetEncoding(section); //save to disk
using (var streamReader = new StreamReader( using (var stream = new FileStream(filePathAndName, FileMode.Create))
section.Body,
encoding,
detectEncodingFromByteOrderMarks: true,
bufferSize: 1024,
leaveOpen: true))
{
// The value length limit is enforced by MultipartBodyLengthLimit
var value = await streamReader.ReadToEndAsync();
if (String.Equals(value, "undefined", StringComparison.OrdinalIgnoreCase))
{ {
value = String.Empty; await section.Body.CopyToAsync(stream);
} }
formAccumulator.Append(key.Value, value); result.UploadedFiles.Add(new UploadedFileInfo()
if (formAccumulator.ValueCount > _defaultFormOptions.ValueCountLimit)
{ {
throw new InvalidDataException($"Form key count limit {_defaultFormOptions.ValueCountLimit} exceeded."); InitialUploadedPathName = filePathAndName,
OriginalFileName = CleanedUploadFileName,
MimeType = section.ContentType
});
}
else if (MultipartRequestHelper.HasFormDataContentDisposition(contentDisposition))
{
// Content-Disposition: form-data; name="key"
//
// value
// Do not limit the key name length here because the
// multipart headers length limit is already in effect.
var key = HeaderUtilities.RemoveQuotes(contentDisposition.Name);
var encoding = GetEncoding(section);
using (var streamReader = new StreamReader(
section.Body,
encoding,
detectEncodingFromByteOrderMarks: true,
bufferSize: 1024,
leaveOpen: true))
{
// The value length limit is enforced by MultipartBodyLengthLimit
var value = await streamReader.ReadToEndAsync();
if (String.Equals(value, "undefined", StringComparison.OrdinalIgnoreCase))
{
value = String.Empty;
}
formAccumulator.Append(key.Value, value);
if (formAccumulator.ValueCount > _defaultFormOptions.ValueCountLimit)
{
throw new InvalidDataException($"Form key count limit {_defaultFormOptions.ValueCountLimit} exceeded.");
}
} }
} }
} }
// Drains any remaining section body that has not been consumed and
// reads the headers for the next section.
section = await reader.ReadNextSectionAsync();
} }
// Drains any remaining section body that has not been consumed and //Get any extra form fields and return them
// reads the headers for the next section. result.FormFieldData = formAccumulator.GetResults();
section = await reader.ReadNextSectionAsync(); return result;
} }
catch (Microsoft.AspNetCore.Http.BadHttpRequestException ex)
{
//most commonly here due to file too large
result.Error = $"Code:{ex.StatusCode}, Error: {ex.Message}";
return result;
//Get any extra form fields and return them }
result.FormFieldData = formAccumulator.GetResults();
return result;
} }
@@ -151,11 +160,13 @@ namespace AyaNova.Api.ControllerHelpers
{ {
public Dictionary<string, Microsoft.Extensions.Primitives.StringValues> FormFieldData { get; set; } public Dictionary<string, Microsoft.Extensions.Primitives.StringValues> FormFieldData { get; set; }
public List<UploadedFileInfo> UploadedFiles { get; set; } public List<UploadedFileInfo> UploadedFiles { get; set; }
public string Error { get; set; }
public ApiUploadedFilesResult() public ApiUploadedFilesResult()
{ {
FormFieldData = new Dictionary<string, Microsoft.Extensions.Primitives.StringValues>(); FormFieldData = new Dictionary<string, Microsoft.Extensions.Primitives.StringValues>();
UploadedFiles = new List<UploadedFileInfo>(); UploadedFiles = new List<UploadedFileInfo>();
Error = null;
} }
} }

43
server/AyaNova/Controllers/AttachmentController.cs
View File

@@ -194,25 +194,18 @@ namespace AyaNova.Api.Controllers
//used to hold extra file data sent by client
// public class UploadFileData
// {
// public string name { get; set; }
// public long lastModified { get; set; }
// }
/// <summary> /// <summary>
/// Upload attachment file /// Upload attachment file
/// /// Max 10GiB total
/// Requires same Authorization roles as object that file is being attached to /// Requires same Authorization roles as object that file is being attached to
/// ///
/// </summary> /// </summary>
/// <returns>NameValue list of filenames and attachment id's</returns> /// <returns>NameValue list of filenames and attachment id's</returns>
[Authorize] [Authorize]
[HttpPost] [HttpPost]
[DisableFormValueModelBinding] [DisableFormValueModelBinding]
// [RequestSizeLimit(10737418241)]//10737418240 = 10gb https://github.com/aspnet/Announcements/issues/267 [RequestSizeLimit(ServerBootConfig.MAX_ATTACHMENT_UPLOAD_BYTES)]
[RequestSizeLimit(1048576)]//1048576 = 1mb for testing
public async Task<IActionResult> UploadAsync() public async Task<IActionResult> UploadAsync()
{ {
//Adapted from the example found here: https://docs.microsoft.com/en-us/aspnet/core/mvc/models/file-uploads#uploading-large-files-with-streaming //Adapted from the example found here: https://docs.microsoft.com/en-us/aspnet/core/mvc/models/file-uploads#uploading-large-files-with-streaming
@@ -228,7 +221,7 @@ namespace AyaNova.Api.Controllers
if (!MultipartRequestHelper.IsMultipartContentType(Request.ContentType)) if (!MultipartRequestHelper.IsMultipartContentType(Request.ContentType))
return BadRequest(new ApiErrorResponse(ApiErrorCode.INVALID_OPERATION, null, $"Expected a multipart request, but got {Request.ContentType}")); return BadRequest(new ApiErrorResponse(ApiErrorCode.INVALID_OPERATION, null, $"Expected a multipart request, but got {Request.ContentType}"));
var uploadFormData = await ApiUploadProcessor.ProcessUploadAsync(HttpContext);
bool badRequest = false; bool badRequest = false;
string AttachToObjectType = string.Empty; string AttachToObjectType = string.Empty;
@@ -237,10 +230,17 @@ namespace AyaNova.Api.Controllers
string Notes = string.Empty; string Notes = string.Empty;
List<UploadFileData> FileData = new List<UploadFileData>(); List<UploadFileData> FileData = new List<UploadFileData>();
if ( var uploadFormData = await ApiUploadProcessor.ProcessUploadAsync(HttpContext);
!uploadFormData.FormFieldData.ContainsKey("FileData") || if (!string.IsNullOrWhiteSpace(uploadFormData.Error))
!uploadFormData.FormFieldData.ContainsKey("AttachToObjectType") || {
!uploadFormData.FormFieldData.ContainsKey("AttachToObjectId")) badRequest = true;
errorMessage = uploadFormData.Error;
}
if (!badRequest
&& (!uploadFormData.FormFieldData.ContainsKey("FileData")
|| !uploadFormData.FormFieldData.ContainsKey("AttachToObjectType")
|| !uploadFormData.FormFieldData.ContainsKey("AttachToObjectId")))
{ {
badRequest = true; badRequest = true;
errorMessage = "Missing one or more required FormFieldData values: AttachToObjectType, AttachToObjectId, FileData"; errorMessage = "Missing one or more required FormFieldData values: AttachToObjectType, AttachToObjectId, FileData";
@@ -314,8 +314,17 @@ namespace AyaNova.Api.Controllers
{ {
//delete temp files //delete temp files
ApiUploadProcessor.DeleteTempUploadFile(uploadFormData); ApiUploadProcessor.DeleteTempUploadFile(uploadFormData);
//return bad request //file too large is most likely issue so in that case return this localized properly
return BadRequest(new ApiErrorResponse(ApiErrorCode.VALIDATION_INVALID_VALUE, null, errorMessage)); if (errorMessage.Contains("413"))
{
var TransId = UserTranslationIdFromContext.Id(HttpContext.Items);
return BadRequest(new ApiErrorResponse(
ApiErrorCode.VALIDATION_LENGTH_EXCEEDED,
null,
String.Format(await TranslationBiz.GetTranslationStaticAsync("AyaFileFileTooLarge", TransId, ct), FileUtil.GetBytesReadable(ServerBootConfig.MAX_ATTACHMENT_UPLOAD_BYTES))));
}
else//not too big, something else
return BadRequest(new ApiErrorResponse(ApiErrorCode.VALIDATION_INVALID_VALUE, null, errorMessage));
} }
long UserId = UserIdFromContext.Id(HttpContext.Items); long UserId = UserIdFromContext.Id(HttpContext.Items);

4
server/AyaNova/Controllers/ImportController.cs
View File

@@ -41,13 +41,13 @@ namespace AyaNova.Api.Controllers
/// <summary> /// <summary>
/// Upload and import file /// Upload and import file
/// Max 100mb total /// Max 100MiB total
/// </summary> /// </summary>
/// <returns>Results</returns> /// <returns>Results</returns>
[Authorize] [Authorize]
[HttpPost("upload")] [HttpPost("upload")]
[DisableFormValueModelBinding] [DisableFormValueModelBinding]
[RequestSizeLimit(100000000)]//100mb limit https://github.com/aspnet/Announcements/issues/267 [RequestSizeLimit(AyaNova.Util.ServerBootConfig.MAX_IMPORT_FILE_UPLOAD_BYTES)]
public async Task<IActionResult> UploadAsync() public async Task<IActionResult> UploadAsync()
{ {
//Adapted from the example found here: https://docs.microsoft.com/en-us/aspnet/core/mvc/models/file-uploads#uploading-large-files-with-streaming //Adapted from the example found here: https://docs.microsoft.com/en-us/aspnet/core/mvc/models/file-uploads#uploading-large-files-with-streaming

10
server/AyaNova/Controllers/LogoController.cs
View File

@@ -36,7 +36,7 @@ namespace AyaNova.Api.Controllers
private readonly AyContext ct; private readonly AyContext ct;
private readonly ILogger<LogoController> log; private readonly ILogger<LogoController> log;
private readonly ApiServerState serverState; private readonly ApiServerState serverState;
private const int MAXIMUM_LOGO_SIZE = 512000;//We really don't want it too big or it will slow the fuck down for everything // private const int MAXIMUM_LOGO_SIZE = 512000;//We really don't want it too big or it will slow the fuck down for everything
/// <summary> /// <summary>
@@ -114,8 +114,8 @@ namespace AyaNova.Api.Controllers
[Authorize] [Authorize]
[HttpPost("{size}")] [HttpPost("{size}")]
//[DisableFormValueModelBinding] //[DisableFormValueModelBinding]
[RequestSizeLimit(MAXIMUM_LOGO_SIZE)]//currently export file is 200kb * 50 maximum at a time = 15mb https://github.com/aspnet/Announcements/issues/267 [RequestSizeLimit(ServerBootConfig.MAX_LOGO_UPLOAD_BYTES)]
public async Task<IActionResult> UploadAsync([FromRoute] string size)//, List<IFormFile> files /// <param name="files">Logo image file</param> public async Task<IActionResult> UploadAsync([FromRoute] string size)
{ {
//https://docs.microsoft.com/en-us/aspnet/core/mvc/models/file-uploads?view=aspnetcore-3.1#upload-small-files-with-buffered-model-binding-to-a-database //https://docs.microsoft.com/en-us/aspnet/core/mvc/models/file-uploads?view=aspnetcore-3.1#upload-small-files-with-buffered-model-binding-to-a-database
@@ -148,8 +148,8 @@ namespace AyaNova.Api.Controllers
using (var memoryStream = new MemoryStream()) using (var memoryStream = new MemoryStream())
{ {
await file.CopyToAsync(memoryStream); await file.CopyToAsync(memoryStream);
if (memoryStream.Length > MAXIMUM_LOGO_SIZE) if (memoryStream.Length > ServerBootConfig.MAX_LOGO_UPLOAD_BYTES)
return BadRequest(new ApiErrorResponse(ApiErrorCode.VALIDATION_LENGTH_EXCEEDED, null, $"Logo files must be smaller than {MAXIMUM_LOGO_SIZE} maximum")); return BadRequest(new ApiErrorResponse(ApiErrorCode.VALIDATION_LENGTH_EXCEEDED, null, $"Logo files must be smaller than {ServerBootConfig.MAX_LOGO_UPLOAD_BYTES} maximum"));
switch (size) switch (size)
{ {
case "small": case "small":

8
server/AyaNova/util/ServerBootConfig.cs
View File

@@ -12,10 +12,14 @@ namespace AyaNova.Util
internal static class ServerBootConfig internal static class ServerBootConfig
{ {
//############################################################################################################ //############################################################################################################
//STATIC HARD CODED DEFAULTS NOT SET THROUGH CONFIG //STATIC HARD CODED COMPILE TIME DEFAULTS NOT SET THROUGH CONFIG
internal const int FAILED_AUTH_DELAY = 3000;//ms internal const int FAILED_AUTH_DELAY = 3000;//ms
internal const int REPORT_RENDERING_OPERATION_TIMEOUT = 20000;//ms internal const int REPORT_RENDERING_OPERATION_TIMEOUT = 20000;//ms
//############################################################################################################ //UPLOAD LIMITS
internal const long MAX_ATTACHMENT_UPLOAD_BYTES = 1048576;//10737418241=10gb in bytes, 1048576 = 1mb for testing
internal const long MAX_LOGO_UPLOAD_BYTES = 512000;//Max 500 KiB total (512000 bytes)
internal const long MAX_IMPORT_FILE_UPLOAD_BYTES = 104857600;//100mib limit
//############################################################################################################
//############################ //############################
//SEEDING FLAG INTERNAL ONLY //SEEDING FLAG INTERNAL ONLY