From abf1c666aaad0977f65a9925bbd21f114ee21c43 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 7 Dec 2025 21:39:19 +0800 Subject: [PATCH 01/10] HDDS-13081. Add S3 Object tagging tests --- .../s3/awssdk/v2/AbstractS3SDKV2Tests.java | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 119849281acc..c76aad12dfa3 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -111,6 +111,7 @@ import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.GetObjectTaggingRequest; +import software.amazon.awssdk.services.s3.model.GetObjectTaggingResponse; import software.amazon.awssdk.services.s3.model.HeadBucketRequest; import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.HeadObjectResponse; @@ -383,6 +384,108 @@ public void testCopyObject() { assertEquals("\"37b51d194a7513e45b56f6524f2d51f2\"", copyObjectResponse.copyObjectResult().eTag()); } + @Test + public void testPutObjectTagging() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "test content"; + s3Client.createBucket(b -> b.bucket(bucketName)); + + s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName), + RequestBody.fromString(content)); + + List tags = Arrays.asList( + Tag.builder().key("env").value("test").build(), + Tag.builder().key("project").value("ozone").build() + ); + + s3Client.putObjectTagging(b -> b + .bucket(bucketName) + .key(keyName) + .tagging(Tagging.builder().tagSet(tags).build())); + + GetObjectTaggingResponse response = s3Client.getObjectTagging(b -> b + .bucket(bucketName) + .key(keyName)); + + assertEquals(tags.size(), response.tagSet().size()); + Map tagMap = response.tagSet().stream() + .collect(Collectors.toMap(Tag::key, Tag::value)); + assertEquals("test", tagMap.get("env")); + assertEquals("ozone", tagMap.get("project")); + } + + @Test + public void testGetObjectTagging() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "test content"; + s3Client.createBucket(b -> b.bucket(bucketName)); + + s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName), + RequestBody.fromString(content)); + + List tags = Arrays.asList( + Tag.builder().key("department").value("engineering").build(), + Tag.builder().key("status").value("active").build() + ); + + s3Client.putObjectTagging(b -> b + .bucket(bucketName) + .key(keyName) + .tagging(Tagging.builder().tagSet(tags).build())); + + GetObjectTaggingResponse response = s3Client.getObjectTagging(b -> b + .bucket(bucketName) + .key(keyName)); + + assertEquals(tags.size(), response.tagSet().size()); + Map retrievedTags = response.tagSet().stream() + .collect(Collectors.toMap(Tag::key, Tag::value)); + assertEquals("engineering", retrievedTags.get("department")); + assertEquals("active", retrievedTags.get("status")); + } + + @Test + public void testDeleteObjectTagging() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "test content"; + s3Client.createBucket(b -> b.bucket(bucketName)); + + s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName), + RequestBody.fromString(content)); + + List tags = Arrays.asList( + Tag.builder().key("temp").value("data").build() + ); + + s3Client.putObjectTagging(b -> b + .bucket(bucketName) + .key(keyName) + .tagging(Tagging.builder().tagSet(tags).build())); + + GetObjectTaggingResponse beforeDelete = s3Client.getObjectTagging(b -> b + .bucket(bucketName) + .key(keyName)); + assertEquals(1, beforeDelete.tagSet().size()); + + s3Client.deleteObjectTagging(b -> b + .bucket(bucketName) + .key(keyName)); + + GetObjectTaggingResponse afterDelete = s3Client.getObjectTagging(b -> b + .bucket(bucketName) + .key(keyName)); + assertTrue(afterDelete.tagSet().isEmpty()); + } + @Test public void testLowLevelMultipartUpload(@TempDir Path tempDir) throws Exception { final String bucketName = getBucketName(); From d127a98a65fc4e36474ae375599f4bca63485076 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 12 Dec 2025 15:37:07 +0800 Subject: [PATCH 02/10] add error cases test --- .../s3/awssdk/v2/AbstractS3SDKV2Tests.java | 171 +++++++++++++++--- 1 file changed, 141 insertions(+), 30 deletions(-) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index c76aad12dfa3..5ba6259d549d 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -81,6 +81,10 @@ import org.junit.jupiter.api.TestMethodOrder; import org.junit.jupiter.api.function.Executable; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import java.util.stream.Stream; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.sync.RequestBody; @@ -385,7 +389,7 @@ public void testCopyObject() { } @Test - public void testPutObjectTagging() { + public void testPutAndGetObjectTagging() { final String bucketName = getBucketName(); final String keyName = getKeyName(); final String content = "test content"; @@ -406,19 +410,19 @@ public void testPutObjectTagging() { .key(keyName) .tagging(Tagging.builder().tagSet(tags).build())); - GetObjectTaggingResponse response = s3Client.getObjectTagging(b -> b + GetObjectTaggingResponse getResponse = s3Client.getObjectTagging(b -> b .bucket(bucketName) .key(keyName)); - assertEquals(tags.size(), response.tagSet().size()); - Map tagMap = response.tagSet().stream() + assertEquals(tags.size(), getResponse.tagSet().size()); + Map tagMap = getResponse.tagSet().stream() .collect(Collectors.toMap(Tag::key, Tag::value)); assertEquals("test", tagMap.get("env")); assertEquals("ozone", tagMap.get("project")); } @Test - public void testGetObjectTagging() { + public void testDeleteObjectTagging() { final String bucketName = getBucketName(); final String keyName = getKeyName(); final String content = "test content"; @@ -430,8 +434,7 @@ public void testGetObjectTagging() { RequestBody.fromString(content)); List tags = Arrays.asList( - Tag.builder().key("department").value("engineering").build(), - Tag.builder().key("status").value("active").build() + Tag.builder().key("temp").value("data").build() ); s3Client.putObjectTagging(b -> b @@ -439,51 +442,159 @@ public void testGetObjectTagging() { .key(keyName) .tagging(Tagging.builder().tagSet(tags).build())); - GetObjectTaggingResponse response = s3Client.getObjectTagging(b -> b + GetObjectTaggingResponse beforeDelete = s3Client.getObjectTagging(b -> b .bucket(bucketName) .key(keyName)); + assertEquals(1, beforeDelete.tagSet().size()); - assertEquals(tags.size(), response.tagSet().size()); - Map retrievedTags = response.tagSet().stream() - .collect(Collectors.toMap(Tag::key, Tag::value)); - assertEquals("engineering", retrievedTags.get("department")); - assertEquals("active", retrievedTags.get("status")); + s3Client.deleteObjectTagging(b -> b + .bucket(bucketName) + .key(keyName)); + + GetObjectTaggingResponse afterDelete = s3Client.getObjectTagging(b -> b + .bucket(bucketName) + .key(keyName)); + assertTrue(afterDelete.tagSet().isEmpty()); } @Test - public void testDeleteObjectTagging() { + public void testPutObjectTaggingExceedsLimit() { final String bucketName = getBucketName(); final String keyName = getKeyName(); - final String content = "test content"; s3Client.createBucket(b -> b.bucket(bucketName)); + s3Client.putObject(b -> b.bucket(bucketName).key(keyName), + RequestBody.fromString("content")); - s3Client.putObject(b -> b + List tags = new ArrayList<>(); + for (int i = 1; i <= 11; i++) { + tags.add(Tag.builder().key("key" + i).value("value" + i).build()); + } + + S3Exception exception = assertThrows(S3Exception.class, () -> + s3Client.putObjectTagging(b -> b .bucket(bucketName) - .key(keyName), - RequestBody.fromString(content)); + .key(keyName) + .tagging(Tagging.builder().tagSet(tags).build()))); + assertEquals(400, exception.statusCode()); + } - List tags = Arrays.asList( - Tag.builder().key("temp").value("data").build() - ); + @Test + public void testPutObjectTaggingReplacesExistingTags() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + s3Client.putObject(b -> b.bucket(bucketName).key(keyName), + RequestBody.fromString("content")); + List initialTags = Arrays.asList( + Tag.builder().key("tag1").value("value1").build(), + Tag.builder().key("tag2").value("value2").build() + ); s3Client.putObjectTagging(b -> b .bucket(bucketName) .key(keyName) - .tagging(Tagging.builder().tagSet(tags).build())); + .tagging(Tagging.builder().tagSet(initialTags).build())); - GetObjectTaggingResponse beforeDelete = s3Client.getObjectTagging(b -> b + List replacementTags = Arrays.asList( + Tag.builder().key("tag3").value("value3").build() + ); + s3Client.putObjectTagging(b -> b .bucket(bucketName) - .key(keyName)); - assertEquals(1, beforeDelete.tagSet().size()); + .key(keyName) + .tagging(Tagging.builder().tagSet(replacementTags).build())); - s3Client.deleteObjectTagging(b -> b + GetObjectTaggingResponse response = s3Client.getObjectTagging(b -> b .bucket(bucketName) .key(keyName)); + assertEquals(1, response.tagSet().size()); + Map tagMap = response.tagSet().stream() + .collect(Collectors.toMap(Tag::key, Tag::value)); + assertEquals("value3", tagMap.get("tag3")); + assertFalse(tagMap.containsKey("tag1")); + assertFalse(tagMap.containsKey("tag2")); + } - GetObjectTaggingResponse afterDelete = s3Client.getObjectTagging(b -> b - .bucket(bucketName) - .key(keyName)); - assertTrue(afterDelete.tagSet().isEmpty()); + private static String repeatChar(char c, int count) { + StringBuilder sb = new StringBuilder(count); + for (int i = 0; i < count; i++) { + sb.append(c); + } + return sb.toString(); + } + + private static Stream invalidTagConstraintsProvider() { + return Stream.of( + Arguments.of( + Arrays.asList(Tag.builder().key(repeatChar('a', 129)).value("value").build()), + 400 + ), + Arguments.of( + Arrays.asList(Tag.builder().key("valid-key").value(repeatChar('b', 257)).build()), + 400 + ), + Arguments.of( + Arrays.asList(Tag.builder().key("t$ag@#invalid").value("value").build()), + 400 + ), + Arguments.of( + Arrays.asList(Tag.builder().key("aws:test").value("value").build()), + 400 + ) + ); + } + + @ParameterizedTest + @MethodSource("invalidTagConstraintsProvider") + public void testPutObjectTaggingInvalidConstraints(List invalidTags, int expectedStatusCode) { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + s3Client.putObject(b -> b.bucket(bucketName).key(keyName), + RequestBody.fromString("content")); + + S3Exception exception = assertThrows(S3Exception.class, () -> + s3Client.putObjectTagging(b -> b + .bucket(bucketName) + .key(keyName) + .tagging(Tagging.builder().tagSet(invalidTags).build()))); + assertEquals(expectedStatusCode, exception.statusCode()); + } + + @Test + public void testPutAndGetObjectTaggingOnNonExistentObject() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + + List tags = Arrays.asList( + Tag.builder().key("env").value("test").build() + ); + + NoSuchKeyException exception = assertThrows(NoSuchKeyException.class, () -> + s3Client.putObjectTagging(b -> b + .bucket(bucketName) + .key(keyName) + .tagging(Tagging.builder().tagSet(tags).build()))); + assertEquals(404, exception.statusCode()); + + exception = assertThrows(NoSuchKeyException.class, () -> + s3Client.getObjectTagging(b -> b + .bucket(bucketName) + .key(keyName))); + assertEquals(404, exception.statusCode()); + } + + @Test + public void testDeleteObjectTaggingOnNonExistentObject() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + + NoSuchKeyException exception = assertThrows(NoSuchKeyException.class, () -> + s3Client.deleteObjectTagging(b -> b + .bucket(bucketName) + .key(keyName))); + assertEquals(404, exception.statusCode()); } @Test From a31204874a48fc0b0f80c7ca292f359bcdda3670 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 12 Dec 2025 15:40:43 +0800 Subject: [PATCH 03/10] fix checkstyle --- .../apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 5ba6259d549d..ceae656b77a8 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -81,10 +81,10 @@ import org.junit.jupiter.api.TestMethodOrder; import org.junit.jupiter.api.function.Executable; import org.junit.jupiter.api.io.TempDir; +import java.util.stream.Stream; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import java.util.stream.Stream; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.sync.RequestBody; From 010cfe7fc006f691d854804519286b474d3be89e Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 12 Dec 2025 16:19:31 +0800 Subject: [PATCH 04/10] checkstyle --- .../apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index ceae656b77a8..93c8827c21f4 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -50,6 +50,7 @@ import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.xml.bind.DatatypeConverter; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.RandomStringUtils; @@ -81,7 +82,6 @@ import org.junit.jupiter.api.TestMethodOrder; import org.junit.jupiter.api.function.Executable; import org.junit.jupiter.api.io.TempDir; -import java.util.stream.Stream; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; From 746a6801f5f04386a26b5636e75e2904cb59766b Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 15 Dec 2025 17:13:13 +0800 Subject: [PATCH 05/10] fix comment --- .../s3/awssdk/v1/AbstractS3SDKV1Tests.java | 91 ++++++++++++++++++- .../s3/awssdk/v2/AbstractS3SDKV2Tests.java | 54 ++++++----- 2 files changed, 120 insertions(+), 25 deletions(-) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index 016ab60537fb..58eabe229cd7 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -23,6 +23,7 @@ import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; import static org.apache.hadoop.ozone.s3.util.S3Utils.stripQuotes; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; @@ -41,6 +42,7 @@ import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; import com.amazonaws.services.s3.model.CreateBucketRequest; import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; +import com.amazonaws.services.s3.model.GetBucketAclRequest; import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.Grantee; import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; @@ -107,7 +109,9 @@ import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -122,7 +126,6 @@ import org.apache.hadoop.ozone.s3.endpoint.S3Owner; import org.apache.hadoop.ozone.s3.util.S3Consts; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.ozone.test.OzoneTestBase; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Nested; @@ -143,7 +146,7 @@ * */ @TestMethodOrder(MethodOrderer.MethodName.class) -public abstract class AbstractS3SDKV1Tests extends OzoneTestBase { +public abstract class AbstractS3SDKV1Tests { // server-side limitation private static final int MAX_UPLOADS_LIMIT = 1000; @@ -1384,6 +1387,86 @@ public void testPresignedUrlDelete() throws IOException { } } + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class BucketOwnershipLinkBucketTests { + private String nonS3VolumeName; + private String linkBucketName; + private String sourceBucketName; + private String danglingSourceBucketName; + private String danglingLinkBucketName; + private OzoneVolume nonS3Volume; + private OzoneVolume s3Volume; + + @BeforeAll + public void setup() throws Exception { + nonS3VolumeName = randomName("link-vol"); + linkBucketName = randomName("link-bucket"); + sourceBucketName = randomName("source"); + danglingSourceBucketName = randomName("link-source"); + danglingLinkBucketName = randomName("link-bucket-dangling"); + try (OzoneClient ozoneClient = cluster.newClient()) { + ObjectStore store = ozoneClient.getObjectStore(); + store.createVolume(nonS3VolumeName); + nonS3Volume = store.getVolume(nonS3VolumeName); + s3Volume = store.getS3Volume(); + } + } + + @Test + public void setBucketVerificationOnLinkBucket() throws Exception { + nonS3Volume.createBucket(sourceBucketName); + BucketArgs.Builder bb = new BucketArgs.Builder() + .setStorageType(StorageType.DEFAULT) + .setVersioning(false) + .setSourceVolume(nonS3VolumeName) + .setSourceBucket(sourceBucketName); + s3Volume.createBucket(linkBucketName, bb.build()); + + GetBucketAclRequest wrongRequest = new GetBucketAclRequest(linkBucketName) + .withExpectedBucketOwner("wrong-owner"); + AmazonServiceException wrongOwner = assertThrows(AmazonServiceException.class, + () -> s3Client.getBucketAcl(wrongRequest)); + assertEquals(403, wrongOwner.getStatusCode()); + assertEquals("AccessDenied", wrongOwner.getErrorCode()); + + Owner owner = s3Client.getBucketAcl(linkBucketName).getOwner(); + GetBucketAclRequest correctRequest = new GetBucketAclRequest(linkBucketName) + .withExpectedBucketOwner(owner.getDisplayName()); + assertDoesNotThrow(() -> s3Client.getBucketAcl(correctRequest)); + } + + @Test + public void testDanglingBucket() throws Exception { + nonS3Volume.createBucket(danglingSourceBucketName); + BucketArgs.Builder bb = new BucketArgs.Builder() + .setStorageType(StorageType.DEFAULT) + .setVersioning(false) + .setSourceVolume(nonS3VolumeName) + .setSourceBucket(danglingSourceBucketName); + s3Volume.createBucket(danglingLinkBucketName, bb.build()); + + nonS3Volume.deleteBucket(danglingSourceBucketName); + + GetBucketAclRequest wrongRequest = new GetBucketAclRequest(danglingLinkBucketName) + .withExpectedBucketOwner("wrong-owner"); + AmazonServiceException wrongOwner = assertThrows(AmazonServiceException.class, + () -> s3Client.getBucketAcl(wrongRequest)); + assertEquals(403, wrongOwner.getStatusCode()); + assertEquals("AccessDenied", wrongOwner.getErrorCode()); + + Owner owner = s3Client.getBucketAcl(danglingLinkBucketName).getOwner(); + GetBucketAclRequest correctRequest = new GetBucketAclRequest(danglingLinkBucketName) + .withExpectedBucketOwner(owner.getDisplayName()); + assertDoesNotThrow(() -> s3Client.getBucketAcl(correctRequest)); + } + + private String randomName(String prefix) { + return (prefix + "-" + RandomStringUtils.secure().nextAlphanumeric(8)) + .toLowerCase(Locale.ROOT); + } + } + /** * Tests the functionality to create a snapshot of an Ozone bucket and then read files * from the snapshot directory using the S3 SDK. @@ -1461,6 +1544,10 @@ private String getKeyName(String suffix) { return (getTestName() + "key" + suffix).toLowerCase(Locale.ROOT); } + private String getTestName() { + return RandomStringUtils.secure().nextAlphanumeric(8).toLowerCase(Locale.ROOT); + } + private String multipartUpload(String bucketName, String key, File file, long partSize, String contentType, Map userMetadata, List tags) throws Exception { String uploadId = initiateMultipartUpload(bucketName, key, contentType, userMetadata, tags); diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 93c8827c21f4..d2a683f07d14 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -60,7 +60,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -475,7 +474,7 @@ public void testPutObjectTaggingExceedsLimit() { .bucket(bucketName) .key(keyName) .tagging(Tagging.builder().tagSet(tags).build()))); - assertEquals(400, exception.statusCode()); + assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, exception.statusCode()); } @Test @@ -575,13 +574,13 @@ public void testPutAndGetObjectTaggingOnNonExistentObject() { .bucket(bucketName) .key(keyName) .tagging(Tagging.builder().tagSet(tags).build()))); - assertEquals(404, exception.statusCode()); + assertEquals(HttpURLConnection.HTTP_NOT_FOUND, exception.statusCode()); exception = assertThrows(NoSuchKeyException.class, () -> s3Client.getObjectTagging(b -> b .bucket(bucketName) .key(keyName))); - assertEquals(404, exception.statusCode()); + assertEquals(HttpURLConnection.HTTP_NOT_FOUND, exception.statusCode()); } @Test @@ -594,7 +593,7 @@ public void testDeleteObjectTaggingOnNonExistentObject() { s3Client.deleteObjectTagging(b -> b .bucket(bucketName) .key(keyName))); - assertEquals(404, exception.statusCode()); + assertEquals(HttpURLConnection.HTTP_NOT_FOUND, exception.statusCode()); } @Test @@ -1885,29 +1884,35 @@ public void testCompleteMultipartUpload() { @Nested @TestInstance(TestInstance.Lifecycle.PER_CLASS) class LinkBucketTests { - private static final String NON_S3_VOLUME_NAME = "link-bucket-volume"; + private String nonS3VolumeName; + private String linkBucketName; + private String danglingSourceBucketName; + private String danglingLinkBucketName; private OzoneVolume nonS3Volume; private OzoneVolume s3Volume; @BeforeAll public void setup() throws Exception { try (OzoneClient ozoneClient = cluster.newClient()) { - ozoneClient.getObjectStore().createVolume(NON_S3_VOLUME_NAME); - nonS3Volume = ozoneClient.getObjectStore().getVolume(NON_S3_VOLUME_NAME); + nonS3VolumeName = randomName("link-vol"); + linkBucketName = randomName("link-bucket"); + danglingSourceBucketName = randomName("link-source"); + danglingLinkBucketName = randomName("link-bucket-dangling"); + ozoneClient.getObjectStore().createVolume(nonS3VolumeName); + nonS3Volume = ozoneClient.getObjectStore().getVolume(nonS3VolumeName); s3Volume = ozoneClient.getObjectStore().getS3Volume(); } } @Test public void setBucketVerificationOnLinkBucket() throws Exception { - // create link bucket - String linkBucketName = "link-bucket"; - nonS3Volume.createBucket(OzoneConsts.BUCKET); + String sourceBucketName = randomName("source"); + nonS3Volume.createBucket(sourceBucketName); BucketArgs.Builder bb = new BucketArgs.Builder() .setStorageType(StorageType.DEFAULT) .setVersioning(false) - .setSourceVolume(NON_S3_VOLUME_NAME) - .setSourceBucket(OzoneConsts.BUCKET); + .setSourceVolume(nonS3VolumeName) + .setSourceBucket(sourceBucketName); s3Volume.createBucket(linkBucketName, bb.build()); GetBucketAclRequest wrongRequest = GetBucketAclRequest.builder() @@ -1930,36 +1935,39 @@ public void setBucketVerificationOnLinkBucket() throws Exception { @Test public void testDanglingBucket() throws Exception { - String sourceBucket = "source-bucket"; - String linkBucket = "link-bucket-dangling"; - nonS3Volume.createBucket(sourceBucket); + nonS3Volume.createBucket(danglingSourceBucketName); BucketArgs.Builder bb = new BucketArgs.Builder() .setStorageType(StorageType.DEFAULT) .setVersioning(false) - .setSourceVolume(NON_S3_VOLUME_NAME) - .setSourceBucket(sourceBucket); - s3Volume.createBucket(linkBucket, bb.build()); + .setSourceVolume(nonS3VolumeName) + .setSourceBucket(danglingSourceBucketName); + s3Volume.createBucket(danglingLinkBucketName, bb.build()); // remove source bucket to make dangling bucket - nonS3Volume.deleteBucket(sourceBucket); + nonS3Volume.deleteBucket(danglingSourceBucketName); GetBucketAclRequest wrongRequest = GetBucketAclRequest.builder() - .bucket(linkBucket) + .bucket(danglingLinkBucketName) .expectedBucketOwner(WRONG_OWNER) .build(); verifyBucketOwnershipVerificationAccessDenied(() -> s3Client.getBucketAcl(wrongRequest)); String owner = s3Client.getBucketAcl(GetBucketAclRequest.builder() - .bucket(linkBucket) + .bucket(danglingLinkBucketName) .build()).owner().displayName(); GetBucketAclRequest correctRequest = GetBucketAclRequest.builder() - .bucket(linkBucket) + .bucket(danglingLinkBucketName) .expectedBucketOwner(owner) .build(); verifyPassBucketOwnershipVerification(() -> s3Client.getBucketAcl(correctRequest)); } + + private String randomName(String prefix) { + return (prefix + "-" + RandomStringUtils.secure().nextAlphanumeric(8)) + .toLowerCase(Locale.ROOT); + } } private void verifyPassBucketOwnershipVerification(Executable function) { From 0888e405b8e0f20082ea9d74b8e3feb154531182 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 15 Dec 2025 20:07:08 +0800 Subject: [PATCH 06/10] fix error --- .../apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index 58eabe229cd7..9771fd4c091d 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -1389,7 +1389,7 @@ public void testPresignedUrlDelete() throws IOException { @Nested @TestInstance(TestInstance.Lifecycle.PER_CLASS) - class BucketOwnershipLinkBucketTests { + static class BucketOwnershipLinkBucketTests { private String nonS3VolumeName; private String linkBucketName; private String sourceBucketName; From 549bb539235b093ed8e8732f75ca9731155277ca Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 21 Dec 2025 00:14:51 +0800 Subject: [PATCH 07/10] consistent LinkBucketTests, remove gettestName keep OzoneTestBase, HTTP --- .../s3/awssdk/v1/AbstractS3SDKV1Tests.java | 16 +++++----------- .../s3/awssdk/v2/AbstractS3SDKV2Tests.java | 17 ++++++++++------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index fbd254cb4a6f..3b44e17113fd 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -96,6 +96,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -107,8 +108,6 @@ import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; - -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.client.BucketArgs; @@ -125,7 +124,6 @@ import org.apache.hadoop.ozone.s3.endpoint.S3Owner; import org.apache.hadoop.ozone.s3.util.S3Consts; import org.apache.hadoop.security.UserGroupInformation; - import org.apache.ozone.test.NonHATests; import org.apache.ozone.test.OzoneTestBase; import org.junit.jupiter.api.BeforeAll; @@ -1049,7 +1047,7 @@ public void testQuotaExceeded() throws IOException { RandomStringUtils.secure().nextAlphanumeric(1024))); assertEquals(ErrorType.Client, ase.getErrorType()); - assertEquals(403, ase.getStatusCode()); + assertEquals(HttpURLConnection.HTTP_FORBIDDEN, ase.getStatusCode()); assertEquals("QuotaExceeded", ase.getErrorCode()); } @@ -1367,7 +1365,7 @@ public void testPresignedUrlDelete() throws IOException { @Nested @TestInstance(TestInstance.Lifecycle.PER_CLASS) - static class BucketOwnershipLinkBucketTests { + class LinkBucketTests { private String nonS3VolumeName; private String linkBucketName; private String sourceBucketName; @@ -1405,7 +1403,7 @@ public void setBucketVerificationOnLinkBucket() throws Exception { .withExpectedBucketOwner("wrong-owner"); AmazonServiceException wrongOwner = assertThrows(AmazonServiceException.class, () -> s3Client.getBucketAcl(wrongRequest)); - assertEquals(403, wrongOwner.getStatusCode()); + assertEquals(HttpURLConnection.HTTP_FORBIDDEN, wrongOwner.getStatusCode()); assertEquals("AccessDenied", wrongOwner.getErrorCode()); Owner owner = s3Client.getBucketAcl(linkBucketName).getOwner(); @@ -1430,7 +1428,7 @@ public void testDanglingBucket() throws Exception { .withExpectedBucketOwner("wrong-owner"); AmazonServiceException wrongOwner = assertThrows(AmazonServiceException.class, () -> s3Client.getBucketAcl(wrongRequest)); - assertEquals(403, wrongOwner.getStatusCode()); + assertEquals(HttpURLConnection.HTTP_FORBIDDEN, wrongOwner.getStatusCode()); assertEquals("AccessDenied", wrongOwner.getErrorCode()); Owner owner = s3Client.getBucketAcl(danglingLinkBucketName).getOwner(); @@ -1522,10 +1520,6 @@ private String getKeyName(String ignored) { return uniqueObjectName(); } - private String getTestName() { - return RandomStringUtils.secure().nextAlphanumeric(8).toLowerCase(Locale.ROOT); - } - private String multipartUpload(String bucketName, String key, File file, long partSize, String contentType, Map userMetadata, List tags) throws Exception { String uploadId = initiateMultipartUpload(bucketName, key, contentType, userMetadata, tags); diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 777dec7739ad..112cc86d7820 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -17,6 +17,8 @@ package org.apache.hadoop.ozone.s3.awssdk.v2; +import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; +import static java.net.HttpURLConnection.HTTP_FORBIDDEN; import static org.apache.hadoop.ozone.OzoneConsts.MB; import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.calculateDigest; import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.createFile; @@ -47,6 +49,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -512,19 +515,19 @@ private static Stream invalidTagConstraintsProvider() { return Stream.of( Arguments.of( Arrays.asList(Tag.builder().key(repeatChar('a', 129)).value("value").build()), - 400 + HTTP_BAD_REQUEST ), Arguments.of( Arrays.asList(Tag.builder().key("valid-key").value(repeatChar('b', 257)).build()), - 400 + HTTP_BAD_REQUEST ), Arguments.of( Arrays.asList(Tag.builder().key("t$ag@#invalid").value("value").build()), - 400 + HTTP_BAD_REQUEST ), Arguments.of( Arrays.asList(Tag.builder().key("aws:test").value("value").build()), - 400 + HTTP_BAD_REQUEST ) ); } @@ -1471,7 +1474,7 @@ public void testHeadBucket() { .expectedBucketOwner(WRONG_OWNER) .build(); S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.headBucket(wrongRequest)); - assertEquals(403, exception.statusCode()); + assertEquals(HTTP_FORBIDDEN, exception.statusCode()); } @Test @@ -1744,7 +1747,7 @@ public void testHeadKey() { .expectedBucketOwner(WRONG_OWNER) .build(); S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.headObject(wrongRequest)); - assertEquals(403, exception.statusCode()); + assertEquals(HTTP_FORBIDDEN, exception.statusCode()); } @Test @@ -1963,7 +1966,7 @@ private void verifyPassBucketOwnershipVerification(Executable function) { private void verifyBucketOwnershipVerificationAccessDenied(Executable function) { S3Exception exception = assertThrows(S3Exception.class, function); - assertEquals(403, exception.statusCode()); + assertEquals(HTTP_FORBIDDEN, exception.statusCode()); assertEquals("Access Denied", exception.awsErrorDetails().errorCode()); } } From 18e9394abb05385c9ecc5cbe589b0315c552caf3 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 29 Dec 2025 17:44:26 +0800 Subject: [PATCH 08/10] fix bucket owner mismatch --- .../hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index 3b44e17113fd..9b0f860ffc00 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -1404,7 +1404,7 @@ public void setBucketVerificationOnLinkBucket() throws Exception { AmazonServiceException wrongOwner = assertThrows(AmazonServiceException.class, () -> s3Client.getBucketAcl(wrongRequest)); assertEquals(HttpURLConnection.HTTP_FORBIDDEN, wrongOwner.getStatusCode()); - assertEquals("AccessDenied", wrongOwner.getErrorCode()); + assertEquals("Access Denied", wrongOwner.getErrorCode()); Owner owner = s3Client.getBucketAcl(linkBucketName).getOwner(); GetBucketAclRequest correctRequest = new GetBucketAclRequest(linkBucketName) @@ -1429,7 +1429,7 @@ public void testDanglingBucket() throws Exception { AmazonServiceException wrongOwner = assertThrows(AmazonServiceException.class, () -> s3Client.getBucketAcl(wrongRequest)); assertEquals(HttpURLConnection.HTTP_FORBIDDEN, wrongOwner.getStatusCode()); - assertEquals("AccessDenied", wrongOwner.getErrorCode()); + assertEquals("Access Denied", wrongOwner.getErrorCode()); Owner owner = s3Client.getBucketAcl(danglingLinkBucketName).getOwner(); GetBucketAclRequest correctRequest = new GetBucketAclRequest(danglingLinkBucketName) From 6c36a6592dfa1c44ac2ad483c2c6884453687df7 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 30 Dec 2025 11:53:59 +0800 Subject: [PATCH 09/10] replace hardcode and add vi tagging test --- .../s3/awssdk/v1/AbstractS3SDKV1Tests.java | 192 ++++++++++++++++++ .../s3/awssdk/v2/AbstractS3SDKV2Tests.java | 9 +- 2 files changed, 198 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index 9b0f860ffc00..3209e8bb9ee5 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -21,6 +21,9 @@ import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.calculateDigest; import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.createFile; import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; +import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_KEY_LENGTH_LIMIT; +import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_NUM_LIMIT; +import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_VALUE_LENGTH_LIMIT; import static org.apache.hadoop.ozone.s3.util.S3Utils.stripQuotes; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; @@ -101,6 +104,7 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.Stream; import javax.xml.bind.DatatypeConverter; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.RandomStringUtils; @@ -134,6 +138,8 @@ import org.junit.jupiter.api.TestMethodOrder; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; /** @@ -540,6 +546,192 @@ public void testGetObjectWithoutETag() throws Exception { } } + @Test + public void testPutAndGetObjectTagging() throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "test content"; + s3Client.createBucket(bucketName); + s3Client.putObject(bucketName, keyName, content); + + Map tags = new HashMap<>(); + tags.put("env", "test"); + tags.put("project", "ozone"); + + try (OzoneClient ozoneClient = cluster.newClient()) { + ObjectStore store = ozoneClient.getObjectStore(); + OzoneVolume volume = store.getS3Volume(); + OzoneBucket bucket = volume.getBucket(bucketName); + bucket.putObjectTagging(keyName, tags); + + Map retrievedTags = bucket.getObjectTagging(keyName); + assertEquals(tags.size(), retrievedTags.size()); + assertEquals("test", retrievedTags.get("env")); + assertEquals("ozone", retrievedTags.get("project")); + } + } + + @Test + public void testDeleteObjectTagging() throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "test content"; + s3Client.createBucket(bucketName); + s3Client.putObject(bucketName, keyName, content); + + Map tags = new HashMap<>(); + tags.put("temp", "data"); + + try (OzoneClient ozoneClient = cluster.newClient()) { + ObjectStore store = ozoneClient.getObjectStore(); + OzoneVolume volume = store.getS3Volume(); + OzoneBucket bucket = volume.getBucket(bucketName); + bucket.putObjectTagging(keyName, tags); + + Map beforeDelete = bucket.getObjectTagging(keyName); + assertEquals(1, beforeDelete.size()); + + bucket.deleteObjectTagging(keyName); + + Map afterDelete = bucket.getObjectTagging(keyName); + assertTrue(afterDelete.isEmpty()); + } + } + + @Test + public void testPutObjectTaggingExceedsLimit() throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(bucketName); + s3Client.putObject(bucketName, keyName, "content"); + + Map tags = new HashMap<>(); + for (int i = 1; i <= TAG_NUM_LIMIT + 1; i++) { + tags.put("key" + i, "value" + i); + } + + try (OzoneClient ozoneClient = cluster.newClient()) { + ObjectStore store = ozoneClient.getObjectStore(); + OzoneVolume volume = store.getS3Volume(); + OzoneBucket bucket = volume.getBucket(bucketName); + assertThrows(Exception.class, () -> bucket.putObjectTagging(keyName, tags)); + } + } + + @Test + public void testPutObjectTaggingReplacesExistingTags() throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(bucketName); + s3Client.putObject(bucketName, keyName, "content"); + + Map initialTags = new HashMap<>(); + initialTags.put("tag1", "value1"); + initialTags.put("tag2", "value2"); + + Map replacementTags = new HashMap<>(); + replacementTags.put("tag3", "value3"); + + try (OzoneClient ozoneClient = cluster.newClient()) { + ObjectStore store = ozoneClient.getObjectStore(); + OzoneVolume volume = store.getS3Volume(); + OzoneBucket bucket = volume.getBucket(bucketName); + bucket.putObjectTagging(keyName, initialTags); + bucket.putObjectTagging(keyName, replacementTags); + + Map retrievedTags = bucket.getObjectTagging(keyName); + assertEquals(1, retrievedTags.size()); + assertEquals("value3", retrievedTags.get("tag3")); + assertFalse(retrievedTags.containsKey("tag1")); + assertFalse(retrievedTags.containsKey("tag2")); + } + } + + private static String repeatChar(char c, int count) { + StringBuilder sb = new StringBuilder(count); + for (int i = 0; i < count; i++) { + sb.append(c); + } + return sb.toString(); + } + + private static Stream invalidTagConstraintsProvider() { + return Stream.of( + Arguments.of( + repeatChar('a', TAG_KEY_LENGTH_LIMIT + 1), + "value" + ), + Arguments.of( + "valid-key", + repeatChar('b', TAG_VALUE_LENGTH_LIMIT + 1) + ), + Arguments.of( + "t$ag@#invalid", + "value" + ), + Arguments.of( + "aws:test", + "value" + ) + ); + } + + @ParameterizedTest + @MethodSource("invalidTagConstraintsProvider") + public void testPutObjectTaggingInvalidConstraints(String tagKey, String tagValue) + throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(bucketName); + s3Client.putObject(bucketName, keyName, "content"); + + Map invalidTags = new HashMap<>(); + invalidTags.put(tagKey, tagValue); + + try (OzoneClient ozoneClient = cluster.newClient()) { + ObjectStore store = ozoneClient.getObjectStore(); + OzoneVolume volume = store.getS3Volume(); + OzoneBucket bucket = volume.getBucket(bucketName); + Exception exception = assertThrows(Exception.class, + () -> bucket.putObjectTagging(keyName, invalidTags)); + assertTrue(exception.getMessage().contains("Invalid") || + exception.getMessage().contains("exceed") || + exception.getMessage().contains("aws:")); + } + } + + @Test + public void testPutAndGetObjectTaggingOnNonExistentObject() throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(bucketName); + + Map tags = new HashMap<>(); + tags.put("env", "test"); + + try (OzoneClient ozoneClient = cluster.newClient()) { + ObjectStore store = ozoneClient.getObjectStore(); + OzoneVolume volume = store.getS3Volume(); + OzoneBucket bucket = volume.getBucket(bucketName); + assertThrows(Exception.class, () -> bucket.putObjectTagging(keyName, tags)); + assertThrows(Exception.class, () -> bucket.getObjectTagging(keyName)); + } + } + + @Test + public void testDeleteObjectTaggingOnNonExistentObject() throws Exception { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + s3Client.createBucket(bucketName); + + try (OzoneClient ozoneClient = cluster.newClient()) { + ObjectStore store = ozoneClient.getObjectStore(); + OzoneVolume volume = store.getS3Volume(); + OzoneBucket bucket = volume.getBucket(bucketName); + assertThrows(Exception.class, () -> bucket.deleteObjectTagging(keyName)); + } + } + @Test public void testListObjectsMany() throws Exception { testListObjectsMany(false); diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 112cc86d7820..15034d19bda5 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -22,6 +22,9 @@ import static org.apache.hadoop.ozone.OzoneConsts.MB; import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.calculateDigest; import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.createFile; +import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_KEY_LENGTH_LIMIT; +import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_NUM_LIMIT; +import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_VALUE_LENGTH_LIMIT; import static org.apache.hadoop.ozone.s3.util.S3Utils.stripQuotes; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; @@ -455,7 +458,7 @@ public void testPutObjectTaggingExceedsLimit() { RequestBody.fromString("content")); List tags = new ArrayList<>(); - for (int i = 1; i <= 11; i++) { + for (int i = 1; i <= TAG_NUM_LIMIT + 1; i++) { tags.add(Tag.builder().key("key" + i).value("value" + i).build()); } @@ -514,11 +517,11 @@ private static String repeatChar(char c, int count) { private static Stream invalidTagConstraintsProvider() { return Stream.of( Arguments.of( - Arrays.asList(Tag.builder().key(repeatChar('a', 129)).value("value").build()), + Arrays.asList(Tag.builder().key(repeatChar('a', TAG_KEY_LENGTH_LIMIT + 1)).value("value").build()), HTTP_BAD_REQUEST ), Arguments.of( - Arrays.asList(Tag.builder().key("valid-key").value(repeatChar('b', 257)).build()), + Arrays.asList(Tag.builder().key("valid-key").value(repeatChar('b', TAG_VALUE_LENGTH_LIMIT + 1)).build()), HTTP_BAD_REQUEST ), Arguments.of( From a3f4db729b9153aa64979a146f789427fa868ba4 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 15 Jan 2026 16:28:50 +0800 Subject: [PATCH 10/10] fix error --- .../s3/tagging/S3PutObjectTaggingRequest.java | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tagging/S3PutObjectTaggingRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tagging/S3PutObjectTaggingRequest.java index cc8064eee017..8df89ed585f4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tagging/S3PutObjectTaggingRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tagging/S3PutObjectTaggingRequest.java @@ -17,12 +17,14 @@ package org.apache.hadoop.ozone.om.request.s3.tagging; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.LeveledResource.BUCKET_LOCK; import java.io.IOException; import java.util.Map; import java.util.Objects; +import java.util.regex.Pattern; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.audit.OMAction; @@ -56,6 +58,59 @@ public class S3PutObjectTaggingRequest extends OMKeyRequest { private static final Logger LOG = LoggerFactory.getLogger(S3PutObjectTaggingRequest.class); + // NOTE: These limits must be kept in sync with + // org.apache.hadoop.ozone.s3.util.S3Consts (S3 Gateway side), + // but OM cannot depend on s3gateway, so we define them here as well. + private static final int TAG_NUM_LIMIT = 10; + private static final int TAG_KEY_LENGTH_LIMIT = 128; + private static final int TAG_VALUE_LENGTH_LIMIT = 256; + private static final String AWS_TAG_PREFIX = "aws:"; + private static final Pattern TAG_REGEX_PATTERN = + Pattern.compile("^([\\p{L}\\p{Z}\\p{N}_.:/=+\\-]*)$"); + + private static void validateTags(Map tags) throws OMException { + if (tags == null || tags.isEmpty()) { + return; + } + + if (tags.size() > TAG_NUM_LIMIT) { + throw new OMException("The number of tags " + tags.size() + + " exceeded the maximum number of tags of " + TAG_NUM_LIMIT, + INVALID_REQUEST); + } + + for (Map.Entry entry : tags.entrySet()) { + String tagKey = entry.getKey(); + String tagValue = entry.getValue() == null ? "" : entry.getValue(); + + if (tagKey.length() > TAG_KEY_LENGTH_LIMIT) { + throw new OMException( + "The tag key exceeds the maximum length of " + TAG_KEY_LENGTH_LIMIT, + INVALID_REQUEST); + } + + if (tagValue.length() > TAG_VALUE_LENGTH_LIMIT) { + throw new OMException( + "The tag value exceeds the maximum length of " + TAG_VALUE_LENGTH_LIMIT, + INVALID_REQUEST); + } + + if (!TAG_REGEX_PATTERN.matcher(tagKey).matches()) { + throw new OMException("Invalid tag key: " + tagKey, INVALID_REQUEST); + } + + if (!TAG_REGEX_PATTERN.matcher(tagValue).matches()) { + throw new OMException("Invalid tag value: " + tagValue, INVALID_REQUEST); + } + + if (tagKey.startsWith(AWS_TAG_PREFIX)) { + throw new OMException( + "Tag key must not start with reserved prefix " + AWS_TAG_PREFIX, + INVALID_REQUEST); + } + } + } + public S3PutObjectTaggingRequest(OMRequest omRequest, BucketLayout bucketLayout) { super(omRequest, bucketLayout); } @@ -127,8 +182,12 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut throw new OMException("Key not found", KEY_NOT_FOUND); } + Map tags = + KeyValueUtil.getFromProtobuf(keyArgs.getTagsList()); + validateTags(tags); + omKeyInfo = omKeyInfo.toBuilder() - .setTags(KeyValueUtil.getFromProtobuf(keyArgs.getTagsList())) + .setTags(tags) .setUpdateID(trxnLogIndex) .build();