From 4b399c6da18832dbb212c417add5930f49530e06 Mon Sep 17 00:00:00 2001 From: Phil Asmar Date: Thu, 9 Oct 2025 10:16:15 -0400 Subject: [PATCH 1/4] Increase multipart upload default part size to 8MB --- .../252dad9f-d2a9-4d49-bff8-000924f0adc3.json | 11 ++++++++++ .../Internal/MultipartUploadCommand.cs | 21 +++++++++---------- 2 files changed, 21 insertions(+), 11 deletions(-) create mode 100644 generator/.DevConfigs/252dad9f-d2a9-4d49-bff8-000924f0adc3.json diff --git a/generator/.DevConfigs/252dad9f-d2a9-4d49-bff8-000924f0adc3.json b/generator/.DevConfigs/252dad9f-d2a9-4d49-bff8-000924f0adc3.json new file mode 100644 index 000000000000..fb2f70752b86 --- /dev/null +++ b/generator/.DevConfigs/252dad9f-d2a9-4d49-bff8-000924f0adc3.json @@ -0,0 +1,11 @@ +{ + "services": [ + { + "serviceName": "S3", + "type": "patch", + "changeLogMessages": [ + "Increasing the default part size for S3 multipart upload from 5MB to 8MB when no part size is specified. This will reduce the number of API calls for multipart uploads." + ] + } + ] +} \ No newline at end of file diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/MultipartUploadCommand.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/MultipartUploadCommand.cs index e31184e6353f..be103558fdf3 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/MultipartUploadCommand.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/MultipartUploadCommand.cs @@ -81,11 +81,12 @@ internal MultipartUploadCommand(IAmazonS3 s3Client, TransferUtilityConfig config this._s3Client = s3Client; this._fileTransporterRequest = fileTransporterRequest; this._contentLength = this._fileTransporterRequest.ContentLength; + + long targetPartSize = fileTransporterRequest.IsSetPartSize() + ? fileTransporterRequest.PartSize + : 8 * 1024 * 1024; // 8MB default (SEP compliant) - if (fileTransporterRequest.IsSetPartSize()) - this._partSize = fileTransporterRequest.PartSize; - else - this._partSize = calculatePartSize(this._contentLength); + this._partSize = calculatePartSize(this._contentLength, targetPartSize); if (fileTransporterRequest.InputStream != null) { @@ -98,15 +99,13 @@ internal MultipartUploadCommand(IAmazonS3 s3Client, TransferUtilityConfig config Logger.DebugFormat("Upload part size {0}.", this._partSize); } - private static long calculatePartSize(long fileSize) + private static long calculatePartSize(long contentLength, long targetPartSize) { - double partSize = Math.Ceiling((double)fileSize / S3Constants.MaxNumberOfParts); - if (partSize < S3Constants.MinPartSize) - { - partSize = S3Constants.MinPartSize; - } + // SEP Formula: Math.Max(targetPartSize, contentLength / 10_000) + long calculatedSize = Math.Max(targetPartSize, contentLength / 10_000); - return (long)partSize; + // Ensure we don't go below S3's minimum part size (5MB) + return Math.Max(calculatedSize, S3Constants.MinPartSize); } private string determineContentType() From 8e4017107caa6120339b4af8a337fc6940d34cc0 Mon Sep 17 00:00:00 2001 From: Phil Asmar Date: Mon, 13 Oct 2025 16:39:22 -0400 Subject: [PATCH 2/4] respond to garretts comments --- .../252dad9f-d2a9-4d49-bff8-000924f0adc3.json | 2 +- .../Internal/MultipartUploadCommand.cs | 5 +--- .../IntegrationTests/TransferUtilityTests.cs | 30 +++++++++++++++++++ 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/generator/.DevConfigs/252dad9f-d2a9-4d49-bff8-000924f0adc3.json b/generator/.DevConfigs/252dad9f-d2a9-4d49-bff8-000924f0adc3.json index fb2f70752b86..402cdd2d7b1c 100644 --- a/generator/.DevConfigs/252dad9f-d2a9-4d49-bff8-000924f0adc3.json +++ b/generator/.DevConfigs/252dad9f-d2a9-4d49-bff8-000924f0adc3.json @@ -2,7 +2,7 @@ "services": [ { "serviceName": "S3", - "type": "patch", + "type": "minor", "changeLogMessages": [ "Increasing the default part size for S3 multipart upload from 5MB to 8MB when no part size is specified. This will reduce the number of API calls for multipart uploads." ] diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/MultipartUploadCommand.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/MultipartUploadCommand.cs index be103558fdf3..28524a530716 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/MultipartUploadCommand.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/MultipartUploadCommand.cs @@ -102,10 +102,7 @@ internal MultipartUploadCommand(IAmazonS3 s3Client, TransferUtilityConfig config private static long calculatePartSize(long contentLength, long targetPartSize) { // SEP Formula: Math.Max(targetPartSize, contentLength / 10_000) - long calculatedSize = Math.Max(targetPartSize, contentLength / 10_000); - - // Ensure we don't go below S3's minimum part size (5MB) - return Math.Max(calculatedSize, S3Constants.MinPartSize); + return Math.Max(targetPartSize, contentLength / S3Constants.MaxNumberOfParts); } private string determineContentType() diff --git a/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs b/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs index abc767a0bdd8..7630d6092b23 100644 --- a/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs +++ b/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs @@ -663,6 +663,36 @@ public void MultipartGetNumberTest() } } + [TestMethod] + [TestCategory("S3")] + public void MultipartValidatePartSize8MbTest() + { + string key = "MultipartValidatePartSizeTest"; + + Upload(key, 20 * MEG_SIZE, null, Client); + + try + { + var objectMetadataResponse = Client.GetObjectMetadata(new GetObjectMetadataRequest + { + BucketName = bucketName, + Key = key, + PartNumber = 1, + }); + + Assert.AreEqual(2, objectMetadataResponse.PartsCount); + Assert.AreEqual(8 * MEG_SIZE, objectMetadataResponse.ContentLength); + } + finally + { + Client.DeleteObject(new DeleteObjectRequest + { + BucketName = bucketName, + Key = key + }); + } + } + void Upload(string fileName, long size, TransferProgressValidator progressValidator, AmazonS3Client client = null) { From 4e8ec4f81d584c0e8d3fbae5de0e46e8056a4336 Mon Sep 17 00:00:00 2001 From: Phil Asmar Date: Mon, 13 Oct 2025 16:41:49 -0400 Subject: [PATCH 3/4] use constant value --- .../S3/Custom/Transfer/Internal/MultipartUploadCommand.cs | 3 +-- sdk/src/Services/S3/Custom/Util/S3Constants.cs | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/MultipartUploadCommand.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/MultipartUploadCommand.cs index 28524a530716..c24da9add8e0 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/MultipartUploadCommand.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/MultipartUploadCommand.cs @@ -84,7 +84,7 @@ internal MultipartUploadCommand(IAmazonS3 s3Client, TransferUtilityConfig config long targetPartSize = fileTransporterRequest.IsSetPartSize() ? fileTransporterRequest.PartSize - : 8 * 1024 * 1024; // 8MB default (SEP compliant) + : S3Constants.DefaultPartSize; this._partSize = calculatePartSize(this._contentLength, targetPartSize); @@ -101,7 +101,6 @@ internal MultipartUploadCommand(IAmazonS3 s3Client, TransferUtilityConfig config private static long calculatePartSize(long contentLength, long targetPartSize) { - // SEP Formula: Math.Max(targetPartSize, contentLength / 10_000) return Math.Max(targetPartSize, contentLength / S3Constants.MaxNumberOfParts); } diff --git a/sdk/src/Services/S3/Custom/Util/S3Constants.cs b/sdk/src/Services/S3/Custom/Util/S3Constants.cs index becbf30231b7..f85ad35e00e5 100644 --- a/sdk/src/Services/S3/Custom/Util/S3Constants.cs +++ b/sdk/src/Services/S3/Custom/Util/S3Constants.cs @@ -33,6 +33,7 @@ internal static class S3Constants internal const int PutObjectDefaultTimeout = 20 * 60 * 1000; internal static readonly long MinPartSize = 5 * (long)Math.Pow(2, 20); + internal static readonly long DefaultPartSize = 8 * (long)Math.Pow(2, 20); internal const int MaxNumberOfParts = 10000; internal const int DefaultBufferSize = 8192; From bb3a12e8488e27a3f3b4dfde302e0f09123508b6 Mon Sep 17 00:00:00 2001 From: Phil Asmar Date: Tue, 14 Oct 2025 11:13:56 -0400 Subject: [PATCH 4/4] fix integ test --- .../IntegrationTests/TransferUtilityTests.cs | 27 ++++++------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs b/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs index 7630d6092b23..cce278d328ae 100644 --- a/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs +++ b/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs @@ -670,27 +670,16 @@ public void MultipartValidatePartSize8MbTest() string key = "MultipartValidatePartSizeTest"; Upload(key, 20 * MEG_SIZE, null, Client); - - try + + var objectMetadataResponse = Client.GetObjectMetadata(new GetObjectMetadataRequest { - var objectMetadataResponse = Client.GetObjectMetadata(new GetObjectMetadataRequest - { - BucketName = bucketName, - Key = key, - PartNumber = 1, - }); + BucketName = bucketName, + Key = key, + PartNumber = 1, + }); - Assert.AreEqual(2, objectMetadataResponse.PartsCount); - Assert.AreEqual(8 * MEG_SIZE, objectMetadataResponse.ContentLength); - } - finally - { - Client.DeleteObject(new DeleteObjectRequest - { - BucketName = bucketName, - Key = key - }); - } + Assert.AreEqual(3, objectMetadataResponse.PartsCount); + Assert.AreEqual(8 * MEG_SIZE, objectMetadataResponse.ContentLength); } void Upload(string fileName, long size,