JCLOUDS-1644: Create AWS S3 buckets with ownership and public access block

apis/s3/src/main/java/org/jclouds/s3/S3Client.java

@@ -66,8 +66,10 @@
 import org.jclouds.s3.binders.BindIterableAsPayloadToDeleteRequest;
 import org.jclouds.s3.binders.BindNoBucketLoggingToXmlPayload;
 import org.jclouds.s3.binders.BindObjectMetadataToRequest;
+import org.jclouds.s3.binders.BindOwnershipControlsToXMLPayload;
 import org.jclouds.s3.binders.BindPartIdsAndETagsToRequest;
 import org.jclouds.s3.binders.BindPayerToXmlPayload;
+import org.jclouds.s3.binders.BindPublicAccessBlockConfigurationToXMLPayload;
 import org.jclouds.s3.binders.BindS3ObjectMetadataToRequest;
 import org.jclouds.s3.domain.AccessControlList;
 import org.jclouds.s3.domain.BucketLogging;
@@ -79,6 +81,7 @@
 import org.jclouds.s3.domain.ListMultipartUploadsResponse;
 import org.jclouds.s3.domain.ObjectMetadata;
 import org.jclouds.s3.domain.Payer;
+import org.jclouds.s3.domain.PublicAccessBlockConfiguration;
 import org.jclouds.s3.domain.S3Object;
 import org.jclouds.s3.fallbacks.FalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists;
 import org.jclouds.s3.filters.RequestAuthorizeSignature;
@@ -813,4 +816,19 @@ ListMultipartUploadsResponse listMultipartUploads(@Bucket @EndpointParam(parser
          @QueryParam("delimiter") @Nullable String delimiter, @QueryParam("max-uploads") @Nullable Integer maxUploads,
          @QueryParam("key-marker") @Nullable String keyMarker, @QueryParam("prefix") @Nullable String prefix,
          @QueryParam("upload-id-marker") @Nullable String uploadIdMarker);
+
+   @Named("PutBucketOwnershipControls")
+   @PUT
+   @Path("/")
+   @QueryParams(keys = "ownershipControls")
+   void putBucketOwnershipControls(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
+         // BucketOwnerPreferred | ObjectWriter | BucketOwnerEnforced
+         @BinderParam(BindOwnershipControlsToXMLPayload.class) String objectOwnership);
+
+   @Named("PutPublicAccessBlock")
+   @PUT
+   @Path("/")
+   @QueryParams(keys = "publicAccessBlock")
+   void putPublicAccessBlock(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
+         @BinderParam(BindPublicAccessBlockConfigurationToXMLPayload.class) PublicAccessBlockConfiguration configuration);
 }

apis/s3/src/main/java/org/jclouds/s3/binders/BindOwnershipControlsToXMLPayload.java

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.s3.binders;
+
+import static org.jclouds.s3.binders.XMLHelper.asString;
+import static org.jclouds.s3.binders.XMLHelper.createDocument;
+import static org.jclouds.s3.binders.XMLHelper.elem;
+import static org.jclouds.s3.binders.XMLHelper.elemWithText;
+
+import jakarta.inject.Singleton;
+import jakarta.ws.rs.core.MediaType;
+import javax.xml.parsers.FactoryConfigurationError;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.TransformerException;
+
+import org.jclouds.http.HttpRequest;
+import org.jclouds.rest.Binder;
+import org.jclouds.s3.reference.S3Constants;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import com.google.common.base.Throwables;
+
+@Singleton
+public final class BindOwnershipControlsToXMLPayload implements Binder {
+   @Override
+   public <R extends HttpRequest> R bindToRequest(R request, Object payload) {
+      String from = (String) payload;
+      try {
+         request.setPayload(generatePayload(from));
+         request.getPayload().getContentMetadata().setContentType(MediaType.TEXT_XML);
+         return request;
+      } catch (Exception e) {
+         Throwables.propagateIfPossible(e);
+         throw new RuntimeException("error transforming acl: " + from, e);
+      }
+   }
+
+   protected String generatePayload(String objectOwnership)
+         throws ParserConfigurationException, FactoryConfigurationError, TransformerException {
+      Document document = createDocument();
+      Element rootNode = elem(document, "OwnershipControls", document);
+      rootNode.setAttribute("xmlns", S3Constants.S3_REST_API_XML_NAMESPACE);
+      Element ruleNode = elem(rootNode, "Rule", document);
+      elemWithText(ruleNode, "ObjectOwnership", objectOwnership, document);
+      return asString(document);
+   }
+}

apis/s3/src/main/java/org/jclouds/s3/binders/BindPublicAccessBlockConfigurationToXMLPayload.java

@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.s3.binders;
+
+import static org.jclouds.s3.binders.XMLHelper.asString;
+import static org.jclouds.s3.binders.XMLHelper.createDocument;
+import static org.jclouds.s3.binders.XMLHelper.elem;
+import static org.jclouds.s3.binders.XMLHelper.elemWithText;
+
+import jakarta.inject.Singleton;
+import jakarta.ws.rs.core.MediaType;
+import javax.xml.parsers.FactoryConfigurationError;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.TransformerException;
+
+import org.jclouds.http.HttpRequest;
+import org.jclouds.rest.Binder;
+import org.jclouds.s3.domain.PublicAccessBlockConfiguration;
+import org.jclouds.s3.reference.S3Constants;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import com.google.common.base.Throwables;
+
+@Singleton
+public final class BindPublicAccessBlockConfigurationToXMLPayload implements Binder {
+   @Override
+   public <R extends HttpRequest> R bindToRequest(R request, Object payload) {
+      PublicAccessBlockConfiguration configuration = (PublicAccessBlockConfiguration) payload;
+      try {
+         request.setPayload(generatePayload(configuration));
+         request.getPayload().getContentMetadata().setContentType(MediaType.TEXT_XML);
+         return request;
+      } catch (Exception e) {
+         Throwables.propagateIfPossible(e);
+         throw new RuntimeException("error transforming configuration: " + configuration, e);
+      }
+   }
+
+   protected String generatePayload(PublicAccessBlockConfiguration configuration)
+         throws ParserConfigurationException, FactoryConfigurationError, TransformerException {
+      Document document = createDocument();
+      Element rootNode = elem(document, "PublicAccessBlockConfiguration", document);
+      rootNode.setAttribute("xmlns", S3Constants.S3_REST_API_XML_NAMESPACE);
+      elemWithText(rootNode, "BlockPublicAcls", String.valueOf(configuration.blockPublicAcls()), document);
+      elemWithText(rootNode, "IgnorePublicAcls", String.valueOf(configuration.ignorePublicAcls()), document);
+      elemWithText(rootNode, "BlockPublicPolicy", String.valueOf(configuration.blockPublicPolicy()), document);
+      elemWithText(rootNode, "RestrictPublicBuckets", String.valueOf(configuration.restrictPublicBuckets()), document);
+      return asString(document);
+   }
+}

apis/s3/src/main/java/org/jclouds/s3/domain/PublicAccessBlockConfiguration.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.s3.domain;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.annotations.Beta;
+
+@AutoValue
+@Beta
+public abstract class PublicAccessBlockConfiguration {
+   public abstract boolean blockPublicAcls();
+   public abstract boolean ignorePublicAcls();
+   public abstract boolean blockPublicPolicy();
+   public abstract boolean restrictPublicBuckets();
+
+   public static PublicAccessBlockConfiguration create(boolean blockPublicAcls, boolean ignorePublicAcls, boolean blockPublicPolicy, boolean restrictPublicBuckets) {
+      return new AutoValue_PublicAccessBlockConfiguration(blockPublicAcls, ignorePublicAcls, blockPublicPolicy, restrictPublicBuckets);
+   }
+}

apis/s3/src/test/java/org/jclouds/s3/S3ClientLiveTest.java

@@ -163,11 +163,16 @@ public void testPutCannedAccessPolicyPublic() throws Exception {
 
    }
 
+   protected void allowPublicReadable(String containerName) {
+   }
+
    @Test(groups = {"fails-on-s3proxy"})
    public void testCopyCannedAccessPolicyPublic() throws Exception {
       String containerName = getContainerName();
       String destinationContainer = getContainerName();
       try {
+         allowPublicReadable(destinationContainer);
+
          addBlobToContainer(containerName, sourceKey);
          validateContent(containerName, sourceKey);
 
@@ -193,6 +198,7 @@ public void testPublicWriteOnObject() throws InterruptedException, ExecutionExce
       final String publicReadWriteObjectKey = "public-read-write-acl";
       final String containerName = getContainerName();
       try {
+         allowPublicReadable(containerName);
          S3Object object = getApi().newS3Object();
          object.getMetadata().setKey(publicReadWriteObjectKey);
          object.setPayload("");
@@ -295,6 +301,7 @@ public void testPublicReadOnObject() throws InterruptedException, ExecutionExcep
       final String publicReadObjectKey = "public-read-acl";
       final String containerName = getContainerName();
       try {
+         allowPublicReadable(containerName);
          S3Object object = getApi().newS3Object();
          object.getMetadata().setKey(publicReadObjectKey);
          object.setPayload("");
@@ -715,6 +722,7 @@ private void addGrantsToACL(AccessControlList acl) {
    public void testUpdateBucketCannedACL() throws Exception {
       String containerName = getContainerName();
       try {
+         allowPublicReadable(containerName);
          getApi().updateBucketCannedACL(containerName, CannedAccessPolicy.PUBLIC_READ);
          AccessControlList acl = getApi().getBucketACL(containerName);
          assertThat(acl.hasPermission(GroupGranteeURI.ALL_USERS, Permission.READ)).isTrue();
@@ -730,6 +738,7 @@ public void testUpdateBucketCannedACL() throws Exception {
    public void testUpdateObjectCannedACL() throws Exception {
       String containerName = getContainerName();
       try {
+         allowPublicReadable(containerName);
          String key = "testUpdateObjectCannedACL";
          S3Object object = getApi().newS3Object();
          object.getMetadata().setKey(key);

apis/s3/src/test/java/org/jclouds/s3/services/BucketsLiveTest.java

@@ -25,14 +25,12 @@
 import static org.jclouds.s3.domain.AccessControlList.Permission.READ_ACP;
 import static org.jclouds.s3.domain.AccessControlList.Permission.WRITE;
 import static org.jclouds.s3.domain.AccessControlList.Permission.WRITE_ACP;
-import static org.jclouds.s3.domain.CannedAccessPolicy.PUBLIC_READ;
 import static org.jclouds.s3.domain.Payer.BUCKET_OWNER;
 import static org.jclouds.s3.domain.Payer.REQUESTER;
 import static org.jclouds.s3.options.ListBucketOptions.Builder.afterMarker;
 import static org.jclouds.s3.options.ListBucketOptions.Builder.delimiter;
 import static org.jclouds.s3.options.ListBucketOptions.Builder.maxResults;
 import static org.jclouds.s3.options.ListBucketOptions.Builder.withPrefix;
-import static org.jclouds.s3.options.PutBucketOptions.Builder.withBucketAcl;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotNull;
 import static org.testng.Assert.assertNull;
@@ -53,6 +51,7 @@
 import org.jclouds.s3.domain.AccessControlList.Grant;
 import org.jclouds.s3.domain.BucketLogging;
 import org.jclouds.s3.domain.BucketMetadata;
+import org.jclouds.s3.domain.CannedAccessPolicy;
 import org.jclouds.s3.domain.ListBucketResponse;
 import org.jclouds.s3.domain.S3Object;
 import org.jclouds.util.Strings2;
@@ -154,7 +153,9 @@ private void addGrantsToACL(AccessControlList acl) {
    public void testPublicReadAccessPolicy() throws Exception {
       String bucketName = getScratchContainerName();
       try {
-         getApi().putBucketInRegion(null, bucketName, withBucketAcl(PUBLIC_READ));
+         getApi().putBucketInRegion(/*region=*/ null, bucketName);
+         allowPublicReadable(bucketName);
+         getApi().updateBucketCannedACL(bucketName, CannedAccessPolicy.PUBLIC_READ);
          AccessControlList acl = getApi().getBucketACL(bucketName);
          assertTrue(acl.hasPermission(ALL_USERS, READ), acl.toString());
          // TODO: I believe that the following should work based on the above acl assertion passing.
@@ -209,11 +210,15 @@ public void run() {
       }
    }
 
+   protected void allowPublicReadable(String containerName) {
+   }
+
    @Test(groups = {"fails-on-s3proxy"})
    public void testBucketLogging() throws Exception {
       final String bucketName = getContainerName();
       final String targetBucket = getContainerName();
       try {
+         allowPublicReadable(targetBucket);
          assertNull(getApi().getBucketLogging(bucketName));
 
          setupAclForBucketLoggingTarget(targetBucket);

blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java

@@ -729,12 +729,16 @@ public void testPutMultipartInputStream() throws Exception {
       testPut(payload, null, new ByteSourcePayload(byteSource), length, new PutOptions().multipart(true));
    }
 
+   protected void allowPublicReadable(String containerName) {
+   }
+
    @Test(groups = { "integration", "live" })
    public void testSetBlobAccess() throws Exception {
       BlobStore blobStore = view.getBlobStore();
       String containerName = getContainerName();
       String blobName = "set-access-blob-name";
       try {
+         allowPublicReadable(containerName);
          addBlobToContainer(containerName, blobName, blobName, MediaType.TEXT_PLAIN);
 
          assertThat(blobStore.getBlobAccess(containerName, blobName)).isEqualTo(BlobAccess.PRIVATE);
@@ -778,6 +782,7 @@ public void testPutBlobAccess() throws Exception {
    public void testPutBlobAccessMultipart() throws Exception {
       BlobStore blobStore = view.getBlobStore();
       String containerName = getContainerName();
+      allowPublicReadable(containerName);
       ByteSource byteSource = TestUtils.randomByteSource().slice(0, getMinimumMultipartBlobSize());
       Payload payload = Payloads.newByteSourcePayload(byteSource);
       payload.getContentMetadata().setContentLength(byteSource.size());

providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java

@@ -29,6 +29,7 @@
 import org.jclouds.aws.s3.blobstore.options.AWSS3PutOptions;
 import org.jclouds.blobstore.BlobStoreContext;
 import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.domain.ContainerAccess;
 import org.jclouds.blobstore.domain.PageSet;
 import org.jclouds.blobstore.domain.StorageMetadata;
 import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
@@ -47,7 +48,9 @@
 import org.jclouds.s3.blobstore.functions.ObjectToBlob;
 import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata;
 import org.jclouds.s3.domain.BucketMetadata;
+import org.jclouds.s3.domain.CannedAccessPolicy;
 import org.jclouds.s3.domain.ObjectMetadata;
+import org.jclouds.s3.domain.PublicAccessBlockConfiguration;
 
 import com.google.common.base.Function;
 import com.google.common.base.Supplier;
@@ -58,6 +61,7 @@
 public class AWSS3BlobStore extends S3BlobStore {
 
    private final BlobToObject blob2Object;
+   private final AWSS3Client awsSync;
 
    @Inject
    AWSS3BlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier<Location> defaultLocation,
@@ -70,6 +74,7 @@ public class AWSS3BlobStore extends S3BlobStore {
       super(context, blobUtils, defaultLocation, locations, slicer, sync, convertBucketsToStorageMetadata,
                container2BucketListOptions, bucket2ResourceList, object2Blob, blob2ObjectGetOptions, blob2Object,
                blob2ObjectMetadata, object2BlobMd, fetchBlobMetadataProvider);
+      this.awsSync = sync;
       this.blob2Object = blob2Object;
    }
 
@@ -102,6 +107,30 @@ public boolean createContainerInLocation(Location location, String container,
          // JCLOUDS-334 for details.
          return false;
       }
+      // AWS blocks creating buckets with public-read canned ACL by default since 25 April 2023.  Instead create a bucket, override the block, and set the ACL.
+      if (options.isPublicRead()) {
+         boolean created = super.createContainerInLocation(location, container, new CreateContainerOptions());
+         if (!created) {
+            return false;
+         }
+         awsSync.putBucketOwnershipControls(container, "ObjectWriter");
+         awsSync.putPublicAccessBlock(container, PublicAccessBlockConfiguration.create(
+               /*blockPublicAcls=*/ false, /*ignorePublicAcls=*/ false, /*blockPublicPolicy=*/ false, /*restrictPublicBuckets=*/ false));
+         awsSync.updateBucketCannedACL(container, CannedAccessPolicy.PUBLIC_READ);
+         return true;
+      }
       return super.createContainerInLocation(location, container, options);
    }
+
+   @Override
+   public void setContainerAccess(String container, ContainerAccess access) {
+      CannedAccessPolicy acl = CannedAccessPolicy.PRIVATE;
+      if (access == ContainerAccess.PUBLIC_READ) {
+         acl = CannedAccessPolicy.PUBLIC_READ;
+         awsSync.putBucketOwnershipControls(container, "ObjectWriter");
+         awsSync.putPublicAccessBlock(container, PublicAccessBlockConfiguration.create(
+               /*blockPublicAcls=*/ false, /*ignorePublicAcls=*/ false, /*blockPublicPolicy=*/ false, /*restrictPublicBuckets=*/ false));
+      }
+      awsSync.updateBucketCannedACL(container, acl);
+   }
 }