From fff40702c6392c0ed5210136b55d2a109c497618 Mon Sep 17 00:00:00 2001 From: hevinhsu Date: Thu, 8 Jan 2026 14:58:57 +0800 Subject: [PATCH 1/5] add integration tests --- .../s3/awssdk/v1/AbstractS3SDKV1Tests.java | 158 ++++++++++++++++++ .../s3/awssdk/v2/AbstractS3SDKV2Tests.java | 146 ++++++++++++++++ 2 files changed, 304 insertions(+) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index c872d19f527c..fd5db0999f0e 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -89,6 +89,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; +import java.util.Base64; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -374,6 +375,163 @@ public void testPutObject() { assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); } + @Test + public void testPutObjectWithMD5Header() throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(bucketName); + + byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); + byte[] md5Bytes = calculateDigest(new ByteArrayInputStream(contentBytes), 0, contentBytes.length); + String md5Base64 = Base64.getEncoder().encodeToString(md5Bytes); + + InputStream is = new ByteArrayInputStream(contentBytes); + ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setContentMD5(md5Base64); + objectMetadata.setContentLength(contentBytes.length); + + PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, objectMetadata); + assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); + + S3Object object = s3Client.getObject(bucketName, keyName); + assertEquals(content.length(), object.getObjectMetadata().getContentLength()); + assertEquals("37b51d194a7513e45b56f6524f2d51f2", object.getObjectMetadata().getETag()); + } + + @Test + public void testPutObjectWithWrongMD5Header() throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(bucketName); + + // Use wrong content to calculate MD5 + byte[] wrongContentBytes = "wrong".getBytes(StandardCharsets.UTF_8); + byte[] wrongMd5Bytes = calculateDigest(new ByteArrayInputStream(wrongContentBytes), 0, wrongContentBytes.length); + String wrongMd5Base64 = Base64.getEncoder().encodeToString(wrongMd5Bytes); + + byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); + InputStream is = new ByteArrayInputStream(contentBytes); + ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setContentMD5(wrongMd5Base64); + objectMetadata.setContentLength(contentBytes.length); + + AmazonServiceException ase = assertThrows(AmazonServiceException.class, + () -> s3Client.putObject(bucketName, keyName, is, objectMetadata)); + + assertEquals(ErrorType.Client, ase.getErrorType()); + assertEquals(400, ase.getStatusCode()); + assertEquals("BadDigest", ase.getErrorCode()); + + // Verify the object was not uploaded + assertFalse(s3Client.doesObjectExist(bucketName, keyName)); + } + + @Test + public void testMultipartUploadWithMD5Header() throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(bucketName); + + // Initiate multipart upload + InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, keyName); + InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); + String uploadId = initResponse.getUploadId(); + + // Prepare part data + String part1Content = "part1data"; + byte[] part1Bytes = part1Content.getBytes(StandardCharsets.UTF_8); + byte[] part1Md5Bytes = calculateDigest(new ByteArrayInputStream(part1Bytes), 0, part1Bytes.length); + String part1Md5Base64 = Base64.getEncoder().encodeToString(part1Md5Bytes); + + // Upload part 1 with MD5 + InputStream part1InputStream = new ByteArrayInputStream(part1Bytes); + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentMD5(part1Md5Base64); + metadata.setContentLength(part1Bytes.length); + + UploadPartRequest uploadRequest = new UploadPartRequest() + .withBucketName(bucketName) + .withKey(keyName) + .withUploadId(uploadId) + .withPartNumber(1) + .withInputStream(part1InputStream) + .withPartSize(part1Bytes.length) + .withObjectMetadata(metadata); + + UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest); + + // Verify ETag + String expectedETag = DatatypeConverter.printHexBinary(part1Md5Bytes).toLowerCase(); + assertEquals(expectedETag, uploadResult.getPartETag().getETag()); + + // Complete multipart upload + List partETags = new ArrayList<>(); + partETags.add(uploadResult.getPartETag()); + + CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest( + bucketName, keyName, uploadId, partETags); + s3Client.completeMultipartUpload(completeRequest); + + // Verify object was uploaded + S3Object object = s3Client.getObject(bucketName, keyName); + try (S3ObjectInputStream s3is = object.getObjectContent(); + ByteArrayOutputStream bos = new ByteArrayOutputStream()) { + IOUtils.copy(s3is, bos); + assertEquals(part1Content, bos.toString("UTF-8")); + } + } + + @Test + public void testMultipartUploadPartWithWrongMD5Header() throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(bucketName); + + // Initiate multipart upload + InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, keyName); + InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); + String uploadId = initResponse.getUploadId(); + + // Prepare part data with wrong MD5 + String partContent = "partdata"; + byte[] partBytes = partContent.getBytes(StandardCharsets.UTF_8); + + byte[] wrongMd5Bytes = calculateDigest( + new ByteArrayInputStream("wrongdata".getBytes(StandardCharsets.UTF_8)), 0, 9); + String wrongMd5Base64 = Base64.getEncoder().encodeToString(wrongMd5Bytes); + + // Upload part with wrong MD5 should fail + InputStream partInputStream = new ByteArrayInputStream(partBytes); + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentMD5(wrongMd5Base64); + metadata.setContentLength(partBytes.length); + + UploadPartRequest uploadRequest = new UploadPartRequest() + .withBucketName(bucketName) + .withKey(keyName) + .withUploadId(uploadId) + .withPartNumber(1) + .withInputStream(partInputStream) + .withPartSize(partBytes.length) + .withObjectMetadata(metadata); + + AmazonServiceException ase = assertThrows(AmazonServiceException.class, + () -> s3Client.uploadPart(uploadRequest)); + + assertEquals(ErrorType.Client, ase.getErrorType()); + assertEquals(400, ase.getStatusCode()); + assertEquals("BadDigest", ase.getErrorCode()); + + // Abort the multipart upload + AbortMultipartUploadRequest abortRequest = new AbortMultipartUploadRequest(bucketName, keyName, uploadId); + s3Client.abortMultipartUpload(abortRequest); + + // Verify object was not created + assertFalse(s3Client.doesObjectExist(bucketName, keyName)); + } + @Test public void testPutDoubleSlashPrefixObject() throws IOException { final String bucketName = getBucketName(); diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 73dac51346da..8bf696f70cf2 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -44,6 +44,7 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; +import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -233,6 +234,151 @@ public void testPutObject() { assertEquals("\"37b51d194a7513e45b56f6524f2d51f2\"", getObjectResponse.eTag()); } + @Test + public void testPutObjectWithMD5Header() throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + + byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); + byte[] md5Bytes = calculateDigest(new ByteArrayInputStream(contentBytes), 0, contentBytes.length); + String md5Base64 = Base64.getEncoder().encodeToString(md5Bytes); + + PutObjectResponse putObjectResponse = s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName) + .contentMD5(md5Base64), + RequestBody.fromString(content)); + + assertEquals("\"37b51d194a7513e45b56f6524f2d51f2\"", putObjectResponse.eTag()); + + ResponseBytes objectBytes = s3Client.getObjectAsBytes( + b -> b.bucket(bucketName).key(keyName) + ); + GetObjectResponse getObjectResponse = objectBytes.response(); + + assertEquals(content, objectBytes.asUtf8String()); + assertEquals("\"37b51d194a7513e45b56f6524f2d51f2\"", getObjectResponse.eTag()); + } + + @Test + public void testPutObjectWithWrongMD5Header() throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + + // Use wrong content to calculate MD5 + byte[] wrongContentBytes = "wrong".getBytes(StandardCharsets.UTF_8); + byte[] wrongMd5Bytes = calculateDigest(new ByteArrayInputStream(wrongContentBytes), 0, wrongContentBytes.length); + String wrongMd5Base64 = Base64.getEncoder().encodeToString(wrongMd5Bytes); + + S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName) + .contentMD5(wrongMd5Base64), + RequestBody.fromString(content))); + + assertEquals(400, exception.statusCode()); + assertEquals("BadDigest", exception.awsErrorDetails().errorCode()); + + // Verify the object was not uploaded + assertThrows(NoSuchKeyException.class, () -> s3Client.headObject(b -> b.bucket(bucketName).key(keyName))); + } + + @Test + public void testMultipartUploadWithMD5Header() throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + + // Initiate multipart upload + CreateMultipartUploadResponse createResponse = s3Client.createMultipartUpload(b -> b + .bucket(bucketName) + .key(keyName)); + String uploadId = createResponse.uploadId(); + + // Prepare part data + String part1Content = "part1data"; + byte[] part1Bytes = part1Content.getBytes(StandardCharsets.UTF_8); + byte[] part1Md5Bytes = calculateDigest(new ByteArrayInputStream(part1Bytes), 0, part1Bytes.length); + String part1Md5Base64 = Base64.getEncoder().encodeToString(part1Md5Bytes); + + // Upload part 1 with MD5 + UploadPartResponse part1Response = s3Client.uploadPart(b -> b + .bucket(bucketName) + .key(keyName) + .uploadId(uploadId) + .partNumber(1) + .contentMD5(part1Md5Base64), + RequestBody.fromBytes(part1Bytes)); + + // Verify ETag + String expectedETag = DatatypeConverter.printHexBinary(part1Md5Bytes).toLowerCase(); + assertEquals(expectedETag, stripQuotes(part1Response.eTag())); + + // Complete multipart upload + CompletedPart completedPart = CompletedPart.builder() + .partNumber(1) + .eTag(part1Response.eTag()) + .build(); + + s3Client.completeMultipartUpload(b -> b + .bucket(bucketName) + .key(keyName) + .uploadId(uploadId) + .multipartUpload(CompletedMultipartUpload.builder().parts(completedPart).build())); + + // Verify object was uploaded + ResponseBytes objectBytes = s3Client.getObjectAsBytes( + b -> b.bucket(bucketName).key(keyName) + ); + assertEquals(part1Content, objectBytes.asUtf8String()); + } + + @Test + public void testMultipartUploadPartWithWrongMD5Header() throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + + // Initiate multipart upload + CreateMultipartUploadResponse createResponse = s3Client.createMultipartUpload(b -> b + .bucket(bucketName) + .key(keyName)); + String uploadId = createResponse.uploadId(); + + // Prepare part data with wrong MD5 + String partContent = "partdata"; + byte[] partBytes = partContent.getBytes(StandardCharsets.UTF_8); + + byte[] wrongMd5Bytes = calculateDigest( + new ByteArrayInputStream("wrongdata".getBytes(StandardCharsets.UTF_8)), 0, 9); + String wrongMd5Base64 = Base64.getEncoder().encodeToString(wrongMd5Bytes); + + // Upload part with wrong MD5 should fail + S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.uploadPart(b -> b + .bucket(bucketName) + .key(keyName) + .uploadId(uploadId) + .partNumber(1) + .contentMD5(wrongMd5Base64), + RequestBody.fromBytes(partBytes))); + + assertEquals(400, exception.statusCode()); + assertEquals("BadDigest", exception.awsErrorDetails().errorCode()); + + // Abort the multipart upload + s3Client.abortMultipartUpload(b -> b + .bucket(bucketName) + .key(keyName) + .uploadId(uploadId)); + + // Verify object was not created + assertThrows(NoSuchKeyException.class, () -> s3Client.headObject(b -> b.bucket(bucketName).key(keyName))); + } + @Test public void testListObjectsMany() throws Exception { testListObjectsMany(false); From 1c794df9b08e20830d86ba0dd668bd0e402839dc Mon Sep 17 00:00:00 2001 From: hevinhsu Date: Thu, 8 Jan 2026 14:59:20 +0800 Subject: [PATCH 2/5] add end to end tests --- .../main/smoketest/s3/MultipartUpload.robot | 52 +++++++++++++++++++ .../src/main/smoketest/s3/objectputget.robot | 23 ++++++++ 2 files changed, 75 insertions(+) diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot index a8d6562bae47..18315fb12fb5 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot @@ -405,3 +405,55 @@ Check Bucket Ownership Verification ${uploadID}= Execute and checkrc echo '${uploadID}' | jq -r '.UploadId' 0 Execute AWSS3APICli with bucket owner check abort-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/mpu/aborttest --upload-id ${uploadID} ${correct_owner} + +Test Multipart Upload Part with Content-MD5 header + # Create test file for multipart upload + Execute echo "Multipart Upload Part Test" > /tmp/mpu_md5testfile + ${md5_hash} = Execute md5sum /tmp/mpu_md5testfile | awk '{print $1}' + ${md5_base64} = Execute openssl dgst -md5 -binary /tmp/mpu_md5testfile | base64 + + # Initialize multipart upload + ${uploadID} = Initiate MPU ${BUCKET} ${PREFIX}/mpu/md5test/key1 + + # Upload part with correct Content-MD5 header + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/mpu/md5test/key1 --part-number 1 --body /tmp/mpu_md5testfile --upload-id ${uploadID} --content-md5 ${md5_base64} + Should contain ${result} ETag + ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' | tr -d '"' 0 + Should Be Equal ${eTag1} ${md5_hash} + + # List parts to verify upload + ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key ${PREFIX}/mpu/md5test/key1 --upload-id ${uploadID} + ${part_etag} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' | tr -d '"' 0 + Should Be Equal ${part_etag} ${md5_hash} + + # Complete the multipart upload + ${parts} = Set Variable {ETag=\"${eTag1}\",PartNumber=1} + Complete MPU ${BUCKET} ${PREFIX}/mpu/md5test/key1 ${uploadID} ${parts} + + # Verify the final object + ${result} = Execute AWSS3APICli get-object --bucket ${BUCKET} --key ${PREFIX}/mpu/md5test/key1 /tmp/mpu_md5testfile.result + Compare files /tmp/mpu_md5testfile /tmp/mpu_md5testfile.result + +Test Multipart Upload Part with wrong Content-MD5 header + # Create test file for multipart upload + Execute echo "Multipart Upload Part Wrong MD5 Test" > /tmp/mpu_md5testfile2 + + # Calculate wrong MD5 (from different content) + ${wrong_md5_hash} = Execute echo -n "wrong content for mpu" | md5sum | awk '{print $1}' + ${wrong_md5_base64} = Execute echo -n "wrong content for mpu" | openssl dgst -md5 -binary | base64 + + # Initialize multipart upload + ${uploadID} = Initiate MPU ${BUCKET} ${PREFIX}/mpu/md5test/key2 + + # Upload part with wrong Content-MD5 header - should fail + ${result} = Execute AWSS3APICli and checkrc upload-part --bucket ${BUCKET} --key ${PREFIX}/mpu/md5test/key2 --part-number 1 --body /tmp/mpu_md5testfile2 --upload-id ${uploadID} --content-md5 ${wrong_md5_base64} 255 + Should contain ${result} BadDigest + + # Verify no parts were uploaded + ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key ${PREFIX}/mpu/md5test/key2 --upload-id ${uploadID} + ${parts_count} = Execute and checkrc echo '${result}' | jq -r '.Parts | length' 0 + Should Be Equal ${parts_count} 0 + + # Abort the multipart upload (cleanup) + Abort MPU ${BUCKET} ${PREFIX}/mpu/md5test/key2 ${uploadID} + diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot index 5053eb4da900..6cafa3513e93 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot @@ -316,3 +316,26 @@ Check Bucket Ownership Verification # create directory Execute touch /tmp/emptyfile Execute AWSS3APICli with bucket owner check put-object --bucket ${BUCKET} --key ${PREFIX}/bucketownercondition/key=value/dir/ --body /tmp/emptyfile ${correct_owner} + +Put object with Content-MD5 header + Execute echo "bar" > /tmp/md5testfile + ${md5_hash} = Execute md5sum /tmp/md5testfile | awk '{print $1}' + ${md5_base64} = Execute openssl dgst -md5 -binary /tmp/md5testfile | base64 + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/md5test/key1 --body /tmp/md5testfile --content-md5 ${md5_base64} + Should contain ${result} ETag + ${etag} = Execute and checkrc echo '${result}' | jq -r '.ETag' | tr -d '"' 0 + Should Be Equal ${etag} ${md5_hash} + ${result} = Execute AWSS3APICli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/md5test/key1 /tmp/md5testfile.result + ${etag} = Execute and checkrc echo '${result}' | jq -r '.ETag' | tr -d '"' 0 + Should Be Equal ${etag} ${md5_hash} + Compare files /tmp/md5testfile /tmp/md5testfile.result + +Put object with wrong Content-MD5 header + Execute echo "bar" > /tmp/md5testfile2 + ${wrong_md5_base64} = Execute echo -n "wrong" | openssl dgst -md5 -binary | base64 + ${result} = Execute AWSS3APICli and checkrc put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/md5test/key2 --body /tmp/md5testfile2 --content-md5 ${wrong_md5_base64} 255 + Should contain ${result} BadDigest + # Verify the object was not uploaded + ${result} = Execute AWSS3APICli and checkrc head-object --bucket ${BUCKET} --key ${PREFIX}/putobject/md5test/key2 255 + Should contain ${result} 404 + From 373fbefef9037f504e33c1a72c2334ce143197e1 Mon Sep 17 00:00:00 2001 From: hevinhsu Date: Thu, 8 Jan 2026 16:41:28 +0800 Subject: [PATCH 3/5] add implementation --- .../ozone/client/io/KeyDataStreamOutput.java | 21 +++++ .../ozone/s3/endpoint/ObjectEndpoint.java | 30 +++++-- .../s3/endpoint/ObjectEndpointStreaming.java | 28 +++++- .../ozone/s3/exception/S3ErrorTable.java | 4 + .../apache/hadoop/ozone/s3/util/S3Consts.java | 2 + .../apache/hadoop/ozone/s3/util/S3Utils.java | 22 +++++ .../hadoop/ozone/client/OzoneBucketStub.java | 86 +++++++++++++++---- .../client/OzoneDataStreamOutputStub.java | 10 +++ .../ozone/s3/endpoint/TestObjectPut.java | 36 ++++++++ .../ozone/s3/endpoint/TestPartUpload.java | 54 +++++++++++- 10 files changed, 268 insertions(+), 25 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java index fffe6e6e81d5..21384d488ddf 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java @@ -111,6 +111,27 @@ public long getClientID() { return clientID; } + @VisibleForTesting + public KeyDataStreamOutput() { + super(null); + this.config = new OzoneClientConfig(); + OmKeyInfo info = new OmKeyInfo.Builder().setKeyName("test").build(); + blockDataStreamOutputEntryPool = + new BlockDataStreamOutputEntryPool( + config, + null, + null, + null, 0, + false, info, + false, + null, + 0L); + + this.writeOffset = 0; + this.clientID = 0L; + this.atomicKeyCreation = false; + } + @SuppressWarnings({"parameternumber", "squid:S00107"}) public KeyDataStreamOutput( OzoneClientConfig config, diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index d33b79761f70..e7d30dd5fd30 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -316,7 +316,7 @@ public Response put( Map tags = getTaggingFromHeaders(getHeaders()); long putLength; - String eTag = null; + final String eTag; if (datastreamEnabled && !enableEC && length > datastreamMinLength) { perf.appendStreamMode(); Pair keyWriteResult = ObjectEndpointStreaming @@ -340,18 +340,29 @@ public Response put( .toLowerCase(); output.getMetadata().put(OzoneConsts.ETAG, eTag); + List> preCommits = new ArrayList<>(); + + String clientContentMD5 = getHeaders().getHeaderString(S3Consts.CHECKSUM_HEADER); + if (clientContentMD5 != null) { + CheckedRunnable checkContentMD5Hook = () -> { + S3Utils.validateContentMD5(clientContentMD5, eTag, keyPath); + }; + preCommits.add(checkContentMD5Hook); + } + // If sha256Digest exists, this request must validate x-amz-content-sha256 MessageDigest sha256Digest = multiDigestInputStream.getMessageDigest(OzoneConsts.FILE_HASH); if (sha256Digest != null) { final String actualSha256 = DatatypeConverter.printHexBinary( sha256Digest.digest()).toLowerCase(); - CheckedRunnable preCommit = () -> { + CheckedRunnable checkSha256Hook = () -> { if (!amzContentSha256Header.equals(actualSha256)) { throw S3ErrorTable.newError(S3ErrorTable.X_AMZ_CONTENT_SHA256_MISMATCH, keyPath); } }; - output.getKeyOutputStream().setPreCommits(Collections.singletonList(preCommit)); + preCommits.add(checkSha256Hook); } + output.getKeyOutputStream().setPreCommits(preCommits); } } getMetrics().incPutKeySuccessLength(putLength); @@ -986,7 +997,7 @@ private Response createMultipartKey(OzoneVolume volume, OzoneBucket ozoneBucket, perf.appendStreamMode(); return ObjectEndpointStreaming .createMultipartKey(ozoneBucket, key, length, partNumber, - uploadID, chunkSize, multiDigestInputStream, perf); + uploadID, chunkSize, multiDigestInputStream, perf, getHeaders()); } // OmMultipartCommitUploadPartInfo can only be gotten after the // OzoneOutputStream is closed, so we need to save the OzoneOutputStream @@ -1066,8 +1077,15 @@ private Response createMultipartKey(OzoneVolume volume, OzoneBucket ozoneBucket, putLength = IOUtils.copyLarge(multiDigestInputStream, ozoneOutputStream, 0, length, new byte[getIOBufferSize(length)]); byte[] digest = multiDigestInputStream.getMessageDigest(OzoneConsts.MD5_HASH).digest(); - ozoneOutputStream.getMetadata() - .put(OzoneConsts.ETAG, DatatypeConverter.printHexBinary(digest).toLowerCase()); + String eTag = DatatypeConverter.printHexBinary(digest).toLowerCase(); + String clientContentMD5 = getHeaders().getHeaderString(S3Consts.CHECKSUM_HEADER); + if (clientContentMD5 != null) { + CheckedRunnable checkContentMD5Hook = () -> { + S3Utils.validateContentMD5(clientContentMD5, eTag, key); + }; + ozoneOutputStream.getKeyOutputStream().setPreCommits(Collections.singletonList(checkContentMD5Hook)); + } + ozoneOutputStream.getMetadata().put(OzoneConsts.ETAG, eTag); outputStream = ozoneOutputStream; } getMetrics().incPutKeySuccessLength(putLength); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java index 8773bf3ca68b..4ab6cf731e40 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java @@ -28,7 +28,9 @@ import java.nio.ByteBuffer; import java.security.DigestInputStream; import java.security.MessageDigest; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Map; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; @@ -45,6 +47,8 @@ import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics; +import org.apache.hadoop.ozone.s3.util.S3Consts; +import org.apache.hadoop.ozone.s3.util.S3Utils; import org.apache.hadoop.util.Time; import org.apache.ratis.util.function.CheckedRunnable; import org.slf4j.Logger; @@ -124,18 +128,29 @@ public static Pair putKeyWithStream( perf.appendMetaLatencyNanos(metadataLatencyNs); ((KeyMetadataAware)streamOutput).getMetadata().put(OzoneConsts.ETAG, eTag); + List> preCommits = new ArrayList<>(); + String clientContentMD5 = headers.getHeaderString(S3Consts.CHECKSUM_HEADER); + if (clientContentMD5 != null) { + CheckedRunnable checkContentMD5Hook = () -> { + S3Utils.validateContentMD5(clientContentMD5, eTag, keyPath); + }; + preCommits.add(checkContentMD5Hook); + } + // If sha256Digest exists, this request must validate x-amz-content-sha256 MessageDigest sha256Digest = body.getMessageDigest(OzoneConsts.FILE_HASH); if (sha256Digest != null) { final String actualSha256 = DatatypeConverter.printHexBinary( sha256Digest.digest()).toLowerCase(); - CheckedRunnable preCommit = () -> { + CheckedRunnable checkSha256Hook = () -> { if (!amzContentSha256Header.equals(actualSha256)) { throw S3ErrorTable.newError(S3ErrorTable.X_AMZ_CONTENT_SHA256_MISMATCH, keyPath); } }; - streamOutput.getKeyDataStreamOutput().setPreCommits(Collections.singletonList(preCommit)); + preCommits.add(checkSha256Hook); } + + streamOutput.getKeyDataStreamOutput().setPreCommits(preCommits); } return Pair.of(eTag, writeLen); } @@ -186,7 +201,7 @@ private static long writeToStreamOutput(OzoneDataStreamOutput streamOutput, @SuppressWarnings("checkstyle:ParameterNumber") public static Response createMultipartKey(OzoneBucket ozoneBucket, String key, long length, int partNumber, String uploadID, int chunkSize, - MultiDigestInputStream body, PerformanceStringBuilder perf) + MultiDigestInputStream body, PerformanceStringBuilder perf, HttpHeaders headers) throws IOException, OS3Exception { long startNanos = Time.monotonicNowNanos(); String eTag; @@ -198,6 +213,13 @@ public static Response createMultipartKey(OzoneBucket ozoneBucket, String key, writeToStreamOutput(streamOutput, body, chunkSize, length); eTag = DatatypeConverter.printHexBinary( body.getMessageDigest(OzoneConsts.MD5_HASH).digest()).toLowerCase(); + String clientContentMD5 = headers.getHeaderString(S3Consts.CHECKSUM_HEADER); + if (clientContentMD5 != null) { + CheckedRunnable checkContentMD5Hook = () -> { + S3Utils.validateContentMD5(clientContentMD5, eTag, key); + }; + streamOutput.getKeyDataStreamOutput().setPreCommits(Collections.singletonList(checkContentMD5Hook)); + } ((KeyMetadataAware)streamOutput).getMetadata().put(OzoneConsts.ETAG, eTag); METRICS.incPutKeySuccessLength(putLength); perf.appendMetaLatencyNanos(metadataLatencyNs); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java index 434087da7462..468d6ea7baa0 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java @@ -164,6 +164,10 @@ public final class S3ErrorTable { "XAmzContentSHA256Mismatch", "The provided 'x-amz-content-sha256' header does " + "not match the computed hash.", HTTP_BAD_REQUEST); + public static final OS3Exception BAD_DIGEST = new OS3Exception( + "BadDigest", "The Content-MD5 or checksum value that you specified did not match what the server received.", + HTTP_BAD_REQUEST); + private static Function generateInternalError = e -> new OS3Exception("InternalError", e.getMessage(), HTTP_INTERNAL_ERROR); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java index 04038bb81cef..797ca1f36712 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java @@ -96,6 +96,8 @@ public final class S3Consts { public static final String EXPECTED_BUCKET_OWNER_HEADER = "x-amz-expected-bucket-owner"; public static final String EXPECTED_SOURCE_BUCKET_OWNER_HEADER = "x-amz-source-expected-bucket-owner"; + public static final String CHECKSUM_HEADER = "Content-MD5"; + //Never Constructed private S3Consts() { diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java index 36c4445470d1..57e8a027edc6 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.s3.util; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.BAD_DIGEST; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_STORAGE_CLASS; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError; import static org.apache.hadoop.ozone.s3.util.S3Consts.AWS_CHUNKED; @@ -32,10 +33,12 @@ import java.net.URLDecoder; import java.net.URLEncoder; import java.util.Arrays; +import java.util.Base64; import java.util.Objects; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; +import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.client.ECReplicationConfig; @@ -222,4 +225,23 @@ public static String wrapInQuotes(String value) { return StringUtils.wrap(value, '\"'); } + /** + * Validates the client provided Content-MD5 against the server's MD5. + * + * @param clientMD5 Base64 encoded MD5 provided by the client + * @param serverMD5 Hexadecimal encoded MD5 stored on the server + * @param resource Resource identifier used when constructing error responses + * @throws OS3Exception if clientMD5 is null or does not match serverMD5 + */ + public static void validateContentMD5(String clientMD5, String serverMD5, String resource) + throws OS3Exception { + if (clientMD5 == null) { + throw newError(BAD_DIGEST, resource); + } + + if (!Hex.encodeHexString(Base64.getDecoder().decode(clientMD5)).equals(serverMD5)) { + throw newError(BAD_DIGEST, resource); + } + } + } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java index 5d060c228c81..8037f65fda1c 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java @@ -47,7 +47,9 @@ import org.apache.hadoop.hdds.scm.storage.ByteBufferStreamOutput; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts.PartInfo; +import org.apache.hadoop.ozone.client.io.KeyDataStreamOutput; import org.apache.hadoop.ozone.client.io.KeyMetadataAware; +import org.apache.hadoop.ozone.client.io.KeyOutputStream; import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; @@ -58,6 +60,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Time; +import org.apache.ratis.util.function.CheckedRunnable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -133,7 +136,7 @@ public OzoneOutputStream createKey(String key, long size, repConfig = rConfig; } ReplicationConfig finalReplicationCon = repConfig; - ByteArrayOutputStream byteArrayOutputStream = + KeyMetadataAwareOutputStream keyOutputStream = new KeyMetadataAwareOutputStream(metadata) { @Override public void close() throws IOException { @@ -145,7 +148,7 @@ public void close() throws IOException { size, System.currentTimeMillis(), System.currentTimeMillis(), - new ArrayList<>(), finalReplicationCon, metadata, null, + new ArrayList<>(), finalReplicationCon, getMetadata(), null, () -> readKey(key), true, UserGroupInformation.getCurrentUser().getShortUserName(), tags @@ -154,7 +157,7 @@ public void close() throws IOException { } }; - return new OzoneOutputStream(byteArrayOutputStream, null); + return new OzoneOutputStream(keyOutputStream, null); } @Override @@ -167,7 +170,7 @@ public OzoneOutputStream rewriteKey(String keyName, long size, long existingKeyG repConfig = rConfig; } ReplicationConfig finalReplicationCon = repConfig; - ByteArrayOutputStream byteArrayOutputStream = + KeyMetadataAwareOutputStream byteArrayOutputStream = new KeyMetadataAwareOutputStream(metadata) { @Override public void close() throws IOException { @@ -204,6 +207,8 @@ public OzoneDataStreamOutput createStreamKey(String key, long size, @Override public void close() throws IOException { + super.close(); + buffer.flip(); byte[] bytes1 = new byte[buffer.remaining()]; buffer.get(bytes1); @@ -258,6 +263,8 @@ public OzoneDataStreamOutput createMultipartStreamKey(String key, @Override public void close() throws IOException { + super.close(); + int position = buffer.position(); buffer.flip(); byte[] bytes = new byte[position]; @@ -438,7 +445,7 @@ public OzoneOutputStream createMultipartKey(String key, long size, if (multipartInfo == null || !multipartInfo.getUploadId().equals(uploadID)) { throw new OMException(ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR); } else { - ByteArrayOutputStream byteArrayOutputStream = + KeyMetadataAwareOutputStream keyOutputStream = new KeyMetadataAwareOutputStream((int) size, new HashMap<>()) { @Override public void close() throws IOException { @@ -451,9 +458,10 @@ public void close() throws IOException { } else { partList.get(key).put(partNumber, part); } + super.close(); } }; - return new OzoneOutputStreamStub(byteArrayOutputStream, key + size); + return new OzoneOutputStreamStub(keyOutputStream, key + size); } } @@ -689,23 +697,59 @@ private void assertDoesNotExist(String keyName) throws OMException { } /** - * ByteArrayOutputStream stub with metadata. + * ByteArrayOutputStream stub with metadata and support for pre-commit hooks. + * This extends KeyOutputStream to allow OzoneOutputStream.getKeyOutputStream() to return a non-null value + * and supports pre-commit hooks like Content-MD5 validation. */ - public static class KeyMetadataAwareOutputStream extends ByteArrayOutputStream - implements KeyMetadataAware { - private Map metadata; + public static class KeyMetadataAwareOutputStream extends KeyOutputStream implements KeyMetadataAware { + private final ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + private final Map metadata; + private List> preCommits = Collections.emptyList(); public KeyMetadataAwareOutputStream(Map metadata) { - super(); + super(null, null); this.metadata = metadata; } public KeyMetadataAwareOutputStream(int size, Map metadata) { - super(size); + super(null, null); this.metadata = metadata; } + @Override + public void write(int b) throws IOException { + buffer.write(b); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + buffer.write(b, off, len); + } + + @Override + public void flush() throws IOException { + buffer.flush(); + } + + @Override + public void close() throws IOException { + // Run pre-commit hooks before closing (e.g., Content-MD5 validation) + for (CheckedRunnable preCommit : preCommits) { + preCommit.run(); + } + buffer.close(); + } + + @Override + public void setPreCommits(List> preCommits) { + this.preCommits = preCommits != null ? preCommits : Collections.emptyList(); + } + + public byte[] toByteArray() { + return buffer.toByteArray(); + } + @Override public Map getMetadata() { return metadata; @@ -713,15 +757,19 @@ public Map getMetadata() { } /** - * ByteBufferOutputStream stub with metadata. + * ByteBufferOutputStream stub with metadata and support for pre-commit hooks. + * This extends KeyDataStreamOutput to allow OzoneDataStreamOutput.getKeyDataStreamOutput() to return a non-null value + * and supports pre-commit hooks like Content-MD5 validation. */ public static class KeyMetadataAwareByteBufferStreamOutput - implements KeyMetadataAware, ByteBufferStreamOutput { + extends KeyDataStreamOutput implements KeyMetadataAware { - private Map metadata; + private final Map metadata; + private List> preCommits = Collections.emptyList(); public KeyMetadataAwareByteBufferStreamOutput( Map metadata) { + super(); this.metadata = metadata; } @@ -738,6 +786,9 @@ public void flush() throws IOException { @Override public void close() throws IOException { + for (CheckedRunnable preCommit : preCommits) { + preCommit.run(); + } } @Override @@ -750,6 +801,11 @@ public void hsync() throws IOException { } + @Override + public void setPreCommits(List> preCommits) { + this.preCommits = preCommits != null ? preCommits : Collections.emptyList(); + } + @Override public Map getMetadata() { return metadata; diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneDataStreamOutputStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneDataStreamOutputStub.java index 6a04c9beb720..c19312692b49 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneDataStreamOutputStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneDataStreamOutputStub.java @@ -21,6 +21,7 @@ import java.nio.ByteBuffer; import org.apache.hadoop.hdds.scm.storage.ByteBufferStreamOutput; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.client.io.KeyDataStreamOutput; import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput; import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo; @@ -65,4 +66,13 @@ public OmMultipartCommitUploadPartInfo getCommitUploadPartInfo() { return closed ? new OmMultipartCommitUploadPartInfo(partName, getMetadata().get(OzoneConsts.ETAG)) : null; } + + @Override + public KeyDataStreamOutput getKeyDataStreamOutput() { + ByteBufferStreamOutput streamOutput = getByteBufStreamOutput(); + if (streamOutput instanceof KeyDataStreamOutput) { + return (KeyDataStreamOutput) streamOutput; + } + return super.getKeyDataStreamOutput(); + } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java index ed6afb29a45e..c8329af41fa5 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java @@ -56,7 +56,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.security.MessageDigest; +import java.util.Base64; import java.util.Map; import java.util.stream.Stream; import javax.ws.rs.core.HttpHeaders; @@ -504,6 +506,40 @@ public void testPutEmptyObject() throws Exception { assertEquals(0, bucket.getKey(KEY_NAME).getDataSize()); } + @Test + public void testPutObjectWithContentMD5() throws Exception { + // GIVEN + byte[] contentBytes = CONTENT.getBytes(StandardCharsets.UTF_8); + byte[] md5Bytes = MessageDigest.getInstance("MD5").digest(contentBytes); + String md5Base64 = Base64.getEncoder().encodeToString(md5Bytes); + + when(headers.getHeaderString("Content-MD5")).thenReturn(md5Base64); + + // WHEN + assertSucceeds(() -> putObject(CONTENT)); + + // THEN + OzoneKeyDetails keyDetails = assertKeyContent(bucket, KEY_NAME, CONTENT); + assertEquals(CONTENT.length(), keyDetails.getDataSize()); + assertNotNull(keyDetails.getMetadata()); + assertThat(keyDetails.getMetadata().get(OzoneConsts.ETAG)).isNotEmpty(); + } + + @Test + public void testPutObjectWithWrongContentMD5() throws Exception { + // GIVEN + byte[] wrongContentBytes = "wrong".getBytes(StandardCharsets.UTF_8); + byte[] wrongMd5Bytes = MessageDigest.getInstance("MD5").digest(wrongContentBytes); + String wrongMd5Base64 = Base64.getEncoder().encodeToString(wrongMd5Bytes); + + // WHEN + when(headers.getHeaderString("Content-MD5")).thenReturn(wrongMd5Base64); + + // WHEN/THEN + OS3Exception ex = assertErrorResponse(S3ErrorTable.BAD_DIGEST, () -> putObject(CONTENT)); + assertThat(ex.getErrorMessage()).contains(S3ErrorTable.BAD_DIGEST.getErrorMessage()); + } + private HttpHeaders newMockHttpHeaders() { HttpHeaders httpHeaders = mock(HttpHeaders.class); when(httpHeaders.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("UNSIGNED-PAYLOAD"); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java index 0ac2508565ce..3af837f9755d 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java @@ -41,7 +41,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.security.MessageDigest; +import java.util.Base64; import java.util.UUID; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; @@ -161,7 +163,7 @@ public void testPartUploadMessageDigestResetDuringException() throws IOException when(rest.getSha256DigestInstance()).thenReturn(sha256Digest); if (enableDataStream) { streaming.when(() -> ObjectEndpointStreaming.createMultipartKey(any(), any(), anyLong(), anyInt(), any(), - anyInt(), any(), any())) + anyInt(), any(), any(), any())) .thenThrow(IOException.class); } else { ioutils.when(() -> IOUtils.copyLarge(any(InputStream.class), any(OutputStream.class), anyLong(), @@ -181,6 +183,56 @@ public void testPartUploadMessageDigestResetDuringException() throws IOException } } + @Test + public void testPartUploadWithContentMD5() throws Exception { + String content = "Multipart Upload Part"; + byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); + byte[] md5Bytes = MessageDigest.getInstance("MD5").digest(contentBytes); + String md5Base64 = Base64.getEncoder().encodeToString(md5Bytes); + + HttpHeaders headersWithMD5 = mock(HttpHeaders.class); + when(headersWithMD5.getHeaderString("Content-MD5")).thenReturn(md5Base64); + when(headersWithMD5.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature"); + when(headersWithMD5.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn("STANDARD"); + + ObjectEndpoint endpoint = EndpointBuilder.newObjectEndpointBuilder() + .setHeaders(headersWithMD5) + .setClient(client) + .build(); + + String uploadID = initiateMultipartUpload(endpoint, OzoneConsts.S3_BUCKET, OzoneConsts.KEY); + + try (Response response = put(endpoint, OzoneConsts.S3_BUCKET, OzoneConsts.KEY, 1, uploadID, content)) { + assertNotNull(response.getHeaderString(OzoneConsts.ETAG)); + assertEquals(200, response.getStatus()); + } + + assertContentLength(uploadID, OzoneConsts.KEY, content.length()); + } + + @Test + public void testPartUploadWithWrongContentMD5() throws Exception { + String content = "Multipart Upload Part"; + byte[] wrongContentBytes = "wrong content".getBytes(StandardCharsets.UTF_8); + byte[] wrongMd5Bytes = MessageDigest.getInstance("MD5").digest(wrongContentBytes); + String wrongMd5Base64 = Base64.getEncoder().encodeToString(wrongMd5Bytes); + + HttpHeaders headersWithWrongMD5 = mock(HttpHeaders.class); + when(headersWithWrongMD5.getHeaderString("Content-MD5")).thenReturn(wrongMd5Base64); + when(headersWithWrongMD5.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature"); + when(headersWithWrongMD5.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn("STANDARD"); + + ObjectEndpoint endpoint = EndpointBuilder.newObjectEndpointBuilder() + .setHeaders(headersWithWrongMD5) + .setClient(client) + .build(); + + String uploadID = initiateMultipartUpload(endpoint, OzoneConsts.S3_BUCKET, OzoneConsts.KEY); + + assertErrorResponse(S3ErrorTable.BAD_DIGEST, + () -> put(endpoint, OzoneConsts.S3_BUCKET, OzoneConsts.KEY, 1, uploadID, content)); + } + private void assertContentLength(String uploadID, String key, long contentLength) throws IOException { OzoneMultipartUploadPartListParts parts = From 15d708ef68c3b430d86e8afa2660130d1d3557e0 Mon Sep 17 00:00:00 2001 From: hevinhsu Date: Fri, 9 Jan 2026 11:39:59 +0800 Subject: [PATCH 4/5] feat: handle IllegalArgumentException when content-md5 is not a base64 string --- .../apache/hadoop/ozone/s3/util/S3Utils.java | 16 ++++++++----- .../ozone/s3/endpoint/TestObjectPut.java | 17 ++++++++++---- .../ozone/s3/endpoint/TestPartUpload.java | 23 +++++++++++++++---- 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java index 57e8a027edc6..39af21bb2758 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java @@ -226,12 +226,12 @@ public static String wrapInQuotes(String value) { } /** - * Validates the client provided Content-MD5 against the server's MD5. + * Validates the Content-MD5 header against the actual MD5 hash. * - * @param clientMD5 Base64 encoded MD5 provided by the client - * @param serverMD5 Hexadecimal encoded MD5 stored on the server - * @param resource Resource identifier used when constructing error responses - * @throws OS3Exception if clientMD5 is null or does not match serverMD5 + * @param clientMD5 the base64-encoded MD5 from Content-MD5 header + * @param serverMD5 the hex-encoded MD5 hash of the data + * @param resource the resource path for error messages + * @throws OS3Exception if the MD5 values do not match */ public static void validateContentMD5(String clientMD5, String serverMD5, String resource) throws OS3Exception { @@ -239,7 +239,11 @@ public static void validateContentMD5(String clientMD5, String serverMD5, String throw newError(BAD_DIGEST, resource); } - if (!Hex.encodeHexString(Base64.getDecoder().decode(clientMD5)).equals(serverMD5)) { + try { + if (!Hex.encodeHexString(Base64.getDecoder().decode(clientMD5)).equals(serverMD5)) { + throw newError(BAD_DIGEST, resource); + } + } catch (IllegalArgumentException ex) { throw newError(BAD_DIGEST, resource); } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java index c8329af41fa5..376d31237a8b 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java @@ -58,6 +58,7 @@ import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.Base64; import java.util.Map; import java.util.stream.Stream; @@ -525,15 +526,23 @@ public void testPutObjectWithContentMD5() throws Exception { assertThat(keyDetails.getMetadata().get(OzoneConsts.ETAG)).isNotEmpty(); } - @Test - public void testPutObjectWithWrongContentMD5() throws Exception { - // GIVEN + + public static Stream wrongContentMD5Provider() throws NoSuchAlgorithmException { byte[] wrongContentBytes = "wrong".getBytes(StandardCharsets.UTF_8); byte[] wrongMd5Bytes = MessageDigest.getInstance("MD5").digest(wrongContentBytes); String wrongMd5Base64 = Base64.getEncoder().encodeToString(wrongMd5Bytes); + return Stream.of( + Arguments.arguments(wrongMd5Base64), + Arguments.arguments("invalid-base64") + ); + } + + @ParameterizedTest + @MethodSource("wrongContentMD5Provider") + public void testPutObjectWithWrongContentMD5(String wrongContentMD5) throws Exception { // WHEN - when(headers.getHeaderString("Content-MD5")).thenReturn(wrongMd5Base64); + when(headers.getHeaderString("Content-MD5")).thenReturn(wrongContentMD5); // WHEN/THEN OS3Exception ex = assertErrorResponse(S3ErrorTable.BAD_DIGEST, () -> putObject(CONTENT)); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java index 3af837f9755d..6885eb6d3b35 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java @@ -43,8 +43,10 @@ import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.Base64; import java.util.UUID; +import java.util.stream.Stream; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; import org.apache.commons.io.IOUtils; @@ -60,6 +62,9 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.Parameter; import org.junit.jupiter.params.ParameterizedClass; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; import org.mockito.MockedStatic; @@ -210,15 +215,23 @@ public void testPartUploadWithContentMD5() throws Exception { assertContentLength(uploadID, OzoneConsts.KEY, content.length()); } - @Test - public void testPartUploadWithWrongContentMD5() throws Exception { - String content = "Multipart Upload Part"; - byte[] wrongContentBytes = "wrong content".getBytes(StandardCharsets.UTF_8); + public static Stream wrongContentMD5Provider() throws NoSuchAlgorithmException { + byte[] wrongContentBytes = "wrong".getBytes(StandardCharsets.UTF_8); byte[] wrongMd5Bytes = MessageDigest.getInstance("MD5").digest(wrongContentBytes); String wrongMd5Base64 = Base64.getEncoder().encodeToString(wrongMd5Bytes); + return Stream.of( + Arguments.arguments(wrongMd5Base64), + Arguments.arguments("invalid-base64") + ); + } + + @ParameterizedTest + @MethodSource("wrongContentMD5Provider") + public void testPartUploadWithWrongContentMD5(String wrongContentMD5) throws Exception { + String content = "Multipart Upload Part"; HttpHeaders headersWithWrongMD5 = mock(HttpHeaders.class); - when(headersWithWrongMD5.getHeaderString("Content-MD5")).thenReturn(wrongMd5Base64); + when(headersWithWrongMD5.getHeaderString("Content-MD5")).thenReturn(wrongContentMD5); when(headersWithWrongMD5.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("mockSignature"); when(headersWithWrongMD5.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn("STANDARD"); From e3b08c91e41c4491c014459de3de25487a836329 Mon Sep 17 00:00:00 2001 From: hevinhsu Date: Fri, 9 Jan 2026 13:29:23 +0800 Subject: [PATCH 5/5] fix checkstyle --- .../java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java index 376d31237a8b..1b39346e1d52 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java @@ -526,7 +526,6 @@ public void testPutObjectWithContentMD5() throws Exception { assertThat(keyDetails.getMetadata().get(OzoneConsts.ETAG)).isNotEmpty(); } - public static Stream wrongContentMD5Provider() throws NoSuchAlgorithmException { byte[] wrongContentBytes = "wrong".getBytes(StandardCharsets.UTF_8); byte[] wrongMd5Bytes = MessageDigest.getInstance("MD5").digest(wrongContentBytes);