Skip to content

Commit

Permalink
JCLOUDS-1644: Create AWS S3 buckets with ownership and public access …
Browse files Browse the repository at this point in the history
…block

AWS changed the defaults when creating buckets to prevent public-read
and other canned ACLs.  Background:
https://stackoverflow.com/a/76102067/2800111
  • Loading branch information
gaul committed Jan 21, 2025
1 parent bdfac92 commit dd6cb51
Show file tree
Hide file tree
Showing 11 changed files with 258 additions and 5 deletions.
18 changes: 18 additions & 0 deletions apis/s3/src/main/java/org/jclouds/s3/S3Client.java
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,10 @@
import org.jclouds.s3.binders.BindIterableAsPayloadToDeleteRequest;
import org.jclouds.s3.binders.BindNoBucketLoggingToXmlPayload;
import org.jclouds.s3.binders.BindObjectMetadataToRequest;
import org.jclouds.s3.binders.BindOwnershipControlsToXMLPayload;
import org.jclouds.s3.binders.BindPartIdsAndETagsToRequest;
import org.jclouds.s3.binders.BindPayerToXmlPayload;
import org.jclouds.s3.binders.BindPublicAccessBlockConfigurationToXMLPayload;
import org.jclouds.s3.binders.BindS3ObjectMetadataToRequest;
import org.jclouds.s3.domain.AccessControlList;
import org.jclouds.s3.domain.BucketLogging;
Expand All @@ -79,6 +81,7 @@
import org.jclouds.s3.domain.ListMultipartUploadsResponse;
import org.jclouds.s3.domain.ObjectMetadata;
import org.jclouds.s3.domain.Payer;
import org.jclouds.s3.domain.PublicAccessBlockConfiguration;
import org.jclouds.s3.domain.S3Object;
import org.jclouds.s3.fallbacks.FalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists;
import org.jclouds.s3.filters.RequestAuthorizeSignature;
Expand Down Expand Up @@ -813,4 +816,19 @@ ListMultipartUploadsResponse listMultipartUploads(@Bucket @EndpointParam(parser
@QueryParam("delimiter") @Nullable String delimiter, @QueryParam("max-uploads") @Nullable Integer maxUploads,
@QueryParam("key-marker") @Nullable String keyMarker, @QueryParam("prefix") @Nullable String prefix,
@QueryParam("upload-id-marker") @Nullable String uploadIdMarker);

@Named("PutBucketOwnershipControls")
@PUT
@Path("/")
@QueryParams(keys = "ownershipControls")
void putBucketOwnershipControls(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
// BucketOwnerPreferred | ObjectWriter | BucketOwnerEnforced
@BinderParam(BindOwnershipControlsToXMLPayload.class) String objectOwnership);

@Named("PutPublicAccessBlock")
@PUT
@Path("/")
@QueryParams(keys = "publicAccessBlock")
void putPublicAccessBlock(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
@BinderParam(BindPublicAccessBlockConfigurationToXMLPayload.class) PublicAccessBlockConfiguration configuration);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.s3.binders;

import static org.jclouds.s3.binders.XMLHelper.asString;
import static org.jclouds.s3.binders.XMLHelper.createDocument;
import static org.jclouds.s3.binders.XMLHelper.elem;
import static org.jclouds.s3.binders.XMLHelper.elemWithText;

import jakarta.inject.Singleton;
import jakarta.ws.rs.core.MediaType;
import javax.xml.parsers.FactoryConfigurationError;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;

import org.jclouds.http.HttpRequest;
import org.jclouds.rest.Binder;
import org.jclouds.s3.reference.S3Constants;
import org.w3c.dom.Document;
import org.w3c.dom.Element;

import com.google.common.base.Throwables;

@Singleton
public final class BindOwnershipControlsToXMLPayload implements Binder {
@Override
public <R extends HttpRequest> R bindToRequest(R request, Object payload) {
String from = (String) payload;
try {
request.setPayload(generatePayload(from));
request.getPayload().getContentMetadata().setContentType(MediaType.TEXT_XML);
return request;
} catch (Exception e) {
Throwables.propagateIfPossible(e);
throw new RuntimeException("error transforming acl: " + from, e);
}
}

protected String generatePayload(String objectOwnership)
throws ParserConfigurationException, FactoryConfigurationError, TransformerException {
Document document = createDocument();
Element rootNode = elem(document, "OwnershipControls", document);
rootNode.setAttribute("xmlns", S3Constants.S3_REST_API_XML_NAMESPACE);
Element ruleNode = elem(rootNode, "Rule", document);
elemWithText(ruleNode, "ObjectOwnership", objectOwnership, document);
return asString(document);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.s3.binders;

import static org.jclouds.s3.binders.XMLHelper.asString;
import static org.jclouds.s3.binders.XMLHelper.createDocument;
import static org.jclouds.s3.binders.XMLHelper.elem;
import static org.jclouds.s3.binders.XMLHelper.elemWithText;

import jakarta.inject.Singleton;
import jakarta.ws.rs.core.MediaType;
import javax.xml.parsers.FactoryConfigurationError;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;

import org.jclouds.http.HttpRequest;
import org.jclouds.rest.Binder;
import org.jclouds.s3.domain.PublicAccessBlockConfiguration;
import org.jclouds.s3.reference.S3Constants;
import org.w3c.dom.Document;
import org.w3c.dom.Element;

import com.google.common.base.Throwables;

@Singleton
public final class BindPublicAccessBlockConfigurationToXMLPayload implements Binder {
@Override
public <R extends HttpRequest> R bindToRequest(R request, Object payload) {
PublicAccessBlockConfiguration configuration = (PublicAccessBlockConfiguration) payload;
try {
request.setPayload(generatePayload(configuration));
request.getPayload().getContentMetadata().setContentType(MediaType.TEXT_XML);
return request;
} catch (Exception e) {
Throwables.propagateIfPossible(e);
throw new RuntimeException("error transforming configuration: " + configuration, e);
}
}

protected String generatePayload(PublicAccessBlockConfiguration configuration)
throws ParserConfigurationException, FactoryConfigurationError, TransformerException {
Document document = createDocument();
Element rootNode = elem(document, "PublicAccessBlockConfiguration", document);
rootNode.setAttribute("xmlns", S3Constants.S3_REST_API_XML_NAMESPACE);
elemWithText(rootNode, "BlockPublicAcls", String.valueOf(configuration.blockPublicAcls()), document);
elemWithText(rootNode, "IgnorePublicAcls", String.valueOf(configuration.ignorePublicAcls()), document);
elemWithText(rootNode, "BlockPublicPolicy", String.valueOf(configuration.blockPublicPolicy()), document);
elemWithText(rootNode, "RestrictPublicBuckets", String.valueOf(configuration.restrictPublicBuckets()), document);
return asString(document);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.s3.domain;

import com.google.auto.value.AutoValue;
import com.google.common.annotations.Beta;

@AutoValue
@Beta
public abstract class PublicAccessBlockConfiguration {
public abstract boolean blockPublicAcls();
public abstract boolean ignorePublicAcls();
public abstract boolean blockPublicPolicy();
public abstract boolean restrictPublicBuckets();

public static PublicAccessBlockConfiguration create(boolean blockPublicAcls, boolean ignorePublicAcls, boolean blockPublicPolicy, boolean restrictPublicBuckets) {
return new AutoValue_PublicAccessBlockConfiguration(blockPublicAcls, ignorePublicAcls, blockPublicPolicy, restrictPublicBuckets);
}
}
9 changes: 9 additions & 0 deletions apis/s3/src/test/java/org/jclouds/s3/S3ClientLiveTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -163,11 +163,16 @@ public void testPutCannedAccessPolicyPublic() throws Exception {

}

protected void allowPublicReadable(String containerName) {
}

@Test(groups = {"fails-on-s3proxy"})
public void testCopyCannedAccessPolicyPublic() throws Exception {
String containerName = getContainerName();
String destinationContainer = getContainerName();
try {
allowPublicReadable(destinationContainer);

addBlobToContainer(containerName, sourceKey);
validateContent(containerName, sourceKey);

Expand All @@ -193,6 +198,7 @@ public void testPublicWriteOnObject() throws InterruptedException, ExecutionExce
final String publicReadWriteObjectKey = "public-read-write-acl";
final String containerName = getContainerName();
try {
allowPublicReadable(containerName);
S3Object object = getApi().newS3Object();
object.getMetadata().setKey(publicReadWriteObjectKey);
object.setPayload("");
Expand Down Expand Up @@ -295,6 +301,7 @@ public void testPublicReadOnObject() throws InterruptedException, ExecutionExcep
final String publicReadObjectKey = "public-read-acl";
final String containerName = getContainerName();
try {
allowPublicReadable(containerName);
S3Object object = getApi().newS3Object();
object.getMetadata().setKey(publicReadObjectKey);
object.setPayload("");
Expand Down Expand Up @@ -715,6 +722,7 @@ private void addGrantsToACL(AccessControlList acl) {
public void testUpdateBucketCannedACL() throws Exception {
String containerName = getContainerName();
try {
allowPublicReadable(containerName);
getApi().updateBucketCannedACL(containerName, CannedAccessPolicy.PUBLIC_READ);
AccessControlList acl = getApi().getBucketACL(containerName);
assertThat(acl.hasPermission(GroupGranteeURI.ALL_USERS, Permission.READ)).isTrue();
Expand All @@ -730,6 +738,7 @@ public void testUpdateBucketCannedACL() throws Exception {
public void testUpdateObjectCannedACL() throws Exception {
String containerName = getContainerName();
try {
allowPublicReadable(containerName);
String key = "testUpdateObjectCannedACL";
S3Object object = getApi().newS3Object();
object.getMetadata().setKey(key);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,12 @@
import static org.jclouds.s3.domain.AccessControlList.Permission.READ_ACP;
import static org.jclouds.s3.domain.AccessControlList.Permission.WRITE;
import static org.jclouds.s3.domain.AccessControlList.Permission.WRITE_ACP;
import static org.jclouds.s3.domain.CannedAccessPolicy.PUBLIC_READ;
import static org.jclouds.s3.domain.Payer.BUCKET_OWNER;
import static org.jclouds.s3.domain.Payer.REQUESTER;
import static org.jclouds.s3.options.ListBucketOptions.Builder.afterMarker;
import static org.jclouds.s3.options.ListBucketOptions.Builder.delimiter;
import static org.jclouds.s3.options.ListBucketOptions.Builder.maxResults;
import static org.jclouds.s3.options.ListBucketOptions.Builder.withPrefix;
import static org.jclouds.s3.options.PutBucketOptions.Builder.withBucketAcl;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
Expand All @@ -53,6 +51,7 @@
import org.jclouds.s3.domain.AccessControlList.Grant;
import org.jclouds.s3.domain.BucketLogging;
import org.jclouds.s3.domain.BucketMetadata;
import org.jclouds.s3.domain.CannedAccessPolicy;
import org.jclouds.s3.domain.ListBucketResponse;
import org.jclouds.s3.domain.S3Object;
import org.jclouds.util.Strings2;
Expand Down Expand Up @@ -154,7 +153,9 @@ private void addGrantsToACL(AccessControlList acl) {
public void testPublicReadAccessPolicy() throws Exception {
String bucketName = getScratchContainerName();
try {
getApi().putBucketInRegion(null, bucketName, withBucketAcl(PUBLIC_READ));
getApi().putBucketInRegion(/*region=*/ null, bucketName);
allowPublicReadable(bucketName);
getApi().updateBucketCannedACL(bucketName, CannedAccessPolicy.PUBLIC_READ);
AccessControlList acl = getApi().getBucketACL(bucketName);
assertTrue(acl.hasPermission(ALL_USERS, READ), acl.toString());
// TODO: I believe that the following should work based on the above acl assertion passing.
Expand Down Expand Up @@ -209,11 +210,15 @@ public void run() {
}
}

protected void allowPublicReadable(String containerName) {
}

@Test(groups = {"fails-on-s3proxy"})
public void testBucketLogging() throws Exception {
final String bucketName = getContainerName();
final String targetBucket = getContainerName();
try {
allowPublicReadable(targetBucket);
assertNull(getApi().getBucketLogging(bucketName));

setupAclForBucketLoggingTarget(targetBucket);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -729,12 +729,16 @@ public void testPutMultipartInputStream() throws Exception {
testPut(payload, null, new ByteSourcePayload(byteSource), length, new PutOptions().multipart(true));
}

protected void allowPublicReadable(String containerName) {
}

@Test(groups = { "integration", "live" })
public void testSetBlobAccess() throws Exception {
BlobStore blobStore = view.getBlobStore();
String containerName = getContainerName();
String blobName = "set-access-blob-name";
try {
allowPublicReadable(containerName);
addBlobToContainer(containerName, blobName, blobName, MediaType.TEXT_PLAIN);

assertThat(blobStore.getBlobAccess(containerName, blobName)).isEqualTo(BlobAccess.PRIVATE);
Expand Down Expand Up @@ -778,6 +782,7 @@ public void testPutBlobAccess() throws Exception {
public void testPutBlobAccessMultipart() throws Exception {
BlobStore blobStore = view.getBlobStore();
String containerName = getContainerName();
allowPublicReadable(containerName);
ByteSource byteSource = TestUtils.randomByteSource().slice(0, getMinimumMultipartBlobSize());
Payload payload = Payloads.newByteSourcePayload(byteSource);
payload.getContentMetadata().setContentLength(byteSource.size());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.jclouds.aws.s3.blobstore.options.AWSS3PutOptions;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.ContainerAccess;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
Expand All @@ -47,7 +48,9 @@
import org.jclouds.s3.blobstore.functions.ObjectToBlob;
import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata;
import org.jclouds.s3.domain.BucketMetadata;
import org.jclouds.s3.domain.CannedAccessPolicy;
import org.jclouds.s3.domain.ObjectMetadata;
import org.jclouds.s3.domain.PublicAccessBlockConfiguration;

import com.google.common.base.Function;
import com.google.common.base.Supplier;
Expand All @@ -58,6 +61,7 @@
public class AWSS3BlobStore extends S3BlobStore {

private final BlobToObject blob2Object;
private final AWSS3Client awsSync;

@Inject
AWSS3BlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier<Location> defaultLocation,
Expand All @@ -70,6 +74,7 @@ public class AWSS3BlobStore extends S3BlobStore {
super(context, blobUtils, defaultLocation, locations, slicer, sync, convertBucketsToStorageMetadata,
container2BucketListOptions, bucket2ResourceList, object2Blob, blob2ObjectGetOptions, blob2Object,
blob2ObjectMetadata, object2BlobMd, fetchBlobMetadataProvider);
this.awsSync = sync;
this.blob2Object = blob2Object;
}

Expand Down Expand Up @@ -102,6 +107,30 @@ public boolean createContainerInLocation(Location location, String container,
// JCLOUDS-334 for details.
return false;
}
// AWS blocks creating buckets with public-read canned ACL by default since 25 April 2023. Instead create a bucket, override the block, and set the ACL.
if (options.isPublicRead()) {
boolean created = super.createContainerInLocation(location, container, new CreateContainerOptions());
if (!created) {
return false;
}
awsSync.putBucketOwnershipControls(container, "ObjectWriter");
awsSync.putPublicAccessBlock(container, PublicAccessBlockConfiguration.create(
/*blockPublicAcls=*/ false, /*ignorePublicAcls=*/ false, /*blockPublicPolicy=*/ false, /*restrictPublicBuckets=*/ false));
awsSync.updateBucketCannedACL(container, CannedAccessPolicy.PUBLIC_READ);
return true;
}
return super.createContainerInLocation(location, container, options);
}

@Override
public void setContainerAccess(String container, ContainerAccess access) {
CannedAccessPolicy acl = CannedAccessPolicy.PRIVATE;
if (access == ContainerAccess.PUBLIC_READ) {
acl = CannedAccessPolicy.PUBLIC_READ;
awsSync.putBucketOwnershipControls(container, "ObjectWriter");
awsSync.putPublicAccessBlock(container, PublicAccessBlockConfiguration.create(
/*blockPublicAcls=*/ false, /*ignorePublicAcls=*/ false, /*blockPublicPolicy=*/ false, /*restrictPublicBuckets=*/ false));
}
awsSync.updateBucketCannedACL(container, acl);
}
}
Loading

0 comments on commit dd6cb51

Please sign in to comment.