Skip to content

Commit

Permalink
#221 fix StaticBucketRouter removes bucketName in the middle of path
Browse files Browse the repository at this point in the history
  • Loading branch information
max402 committed Sep 16, 2024
1 parent 32d36bc commit 4bc43fc
Show file tree
Hide file tree
Showing 10 changed files with 26 additions and 12 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,8 @@ void initDatasafe() {
accessKey(CREDENTIALS),
secretKey(CREDENTIALS)
),
CREDENTIALS,
REGION,
CREDENTIALS,
EXECUTOR
);

Expand All @@ -158,6 +159,7 @@ void initDatasafe() {
acc.getAccessKey(),
acc.getSecretKey()
),
acc.getRegion(),
acc.getBucketName(),
EXECUTOR
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ private static StorageService amazonS3() {
acc.getAccessKey(),
acc.getSecretKey()
),
acc.getRegion(),
// Bucket name is encoded in first path segment
acc.getBucketName(),
ExecutorServiceUtil.submitterExecutesOnStarvationExecutingService()
Expand Down Expand Up @@ -134,6 +135,7 @@ private static S3StorageService getStorageService(String accessKey, String secre

return new S3StorageService(
amazons3,
region,
bucket,
ExecutorServiceUtil
.submitterExecutesOnStarvationExecutingService(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ void testMultiUserStorageUserSetup() {
StorageService directoryStorage = new S3StorageService(
directoryClient,
DIRECTORY_BUCKET.getBucketName(),
REGION,
EXECUTOR
);

Expand Down Expand Up @@ -137,6 +138,7 @@ void testMultiUserStorageUserSetup() {
),
// Bucket name is encoded in first path segment
acc.getBucketName(),
acc.getRegion(),
EXECUTOR
)
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ void init() {
.config(new DefaultDFSConfig(cephMappedUrl, "secret"::toCharArray))
.storage(new S3StorageService(
cephS3,
"",
VERSIONED_BUCKET_NAME,
ExecutorServiceUtil.submitterExecutesOnStarvationExecutingService()))
.build();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@ StorageService clientCredentials(AmazonS3 s3, S3Factory factory, DatasafePropert
ExecutorService executorService = ExecutorServiceUtil.submitterExecutesOnStarvationExecutingService();
S3StorageService basicStorage = new S3StorageService(
s3,
properties.getAmazonRegion(),
properties.getBucketName(),
executorService
);
Expand Down Expand Up @@ -185,6 +186,7 @@ StorageService singleStorageServiceFilesystem(DatasafeProperties properties) {
StorageService singleStorageServiceS3(AmazonS3 s3, DatasafeProperties properties) {
return new S3StorageService(
s3,
properties.getAmazonRegion(),
properties.getBucketName(),
ExecutorServiceUtil.submitterExecutesOnStarvationExecutingService()
);
Expand All @@ -203,7 +205,7 @@ StorageService multiStorageService(DatasafeProperties properties) {
)
);

S3StorageService s3StorageService = new S3StorageService(s3(properties), properties.getBucketName(),
S3StorageService s3StorageService = new S3StorageService(s3(properties), properties.getAmazonRegion(), properties.getBucketName(),
ExecutorServiceUtil.submitterExecutesOnStarvationExecutingService()
);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,7 @@ private static SystemRootAndStorageService useAmazonS3(AmazonS3DFSCredentials df
}
StorageService storageService = new S3StorageService(
amazons3,
amazonS3DFSCredentials.getRegion(),
amazonS3DFSCredentials.getContainer(),
ExecutorServiceUtil
.submitterExecutesOnStarvationExecutingService(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,13 @@ public class S3StorageService implements StorageService {

/**
* @param s3 Connection to S3
* @param bucketName Bucket to use
* @param region Region to use
* @param bucket Bucket to use
* @param executorService Multipart sending threadpool (file chunks are sent in parallel)
*/
public S3StorageService(AmazonS3 s3, String bucketName, ExecutorService executorService) {
public S3StorageService(AmazonS3 s3, String region, String bucket, ExecutorService executorService) {
this.s3 = s3;
this.router = new StaticBucketRouter(bucketName);
this.router = new StaticBucketRouter(region, bucket);
this.executorService = executorService;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
@RequiredArgsConstructor
public class StaticBucketRouter implements BucketRouter {

private final String region;
private final String bucketName;

@Override
Expand All @@ -20,10 +21,11 @@ public String resourceKey(AbsoluteLocation resource) {
UnaryOperator<String> trimStartingSlash = str -> str.replaceFirst("^/", "");

String resourcePath = trimStartingSlash.apply(resource.location().getRawPath());
if (bucketName == null || "".equals(bucketName) || !resourcePath.contains(bucketName)) {
String bucketNameWithRegion = region + "/" + bucketName;
if (bucketName == null || "".equals(bucketName) || !resourcePath.startsWith(bucketNameWithRegion)) {
return resourcePath;
}

return trimStartingSlash.apply(resourcePath.substring(resourcePath.indexOf(bucketName) + bucketName.length()));
return trimStartingSlash.apply(resourcePath.substring(resourcePath.indexOf(bucketNameWithRegion) + bucketNameWithRegion.length()));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ static void beforeAll() {
void init() {
this.storageService = new S3StorageService(
s3,
"eu-central-1",
bucketName,
ExecutorServiceUtil.submitterExecutesOnStarvationExecutingService()
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ protected static StorageDescriptor minio() {
StorageDescriptorName.MINIO,
() -> {
minioStorage.get();
return new S3StorageService(minio, primaryBucket, EXECUTOR_SERVICE);
return new S3StorageService(minio, minioRegion, primaryBucket, EXECUTOR_SERVICE);
},
new Uri("s3://" + primaryBucket + "/" + bucketPath + "/"),
minioAccessKeyID,
Expand All @@ -254,7 +254,7 @@ protected static StorageDescriptor cephVersioned() {
StorageDescriptorName.CEPH,
() -> {
cephStorage.get();
return new S3StorageService(ceph, primaryBucket, EXECUTOR_SERVICE);
return new S3StorageService(ceph, cephRegion, primaryBucket, EXECUTOR_SERVICE);
},
new Uri("s3://" + primaryBucket + "/" + bucketPath + "/"),
cephAccessKeyID,
Expand All @@ -275,10 +275,10 @@ private static boolean skipCeph() {

protected static Function<String, StorageService> storageServiceByBucket() {
if (null == amazonS3) {
return bucketName -> new S3StorageService(minio, bucketName, EXECUTOR_SERVICE);
return bucketName -> new S3StorageService(minio, amazonRegion, bucketName, EXECUTOR_SERVICE);
}

return bucketName -> new S3StorageService(amazonS3, bucketName, EXECUTOR_SERVICE);
return bucketName -> new S3StorageService(amazonS3, amazonRegion, bucketName, EXECUTOR_SERVICE);
}

protected static StorageDescriptor s3() {
Expand All @@ -290,7 +290,7 @@ protected static StorageDescriptor s3() {
StorageDescriptorName.AMAZON,
() -> {
amazonStorage.get();
return new S3StorageService(amazonS3, primaryBucket, EXECUTOR_SERVICE);
return new S3StorageService(amazonS3, amazonRegion, primaryBucket, EXECUTOR_SERVICE);
},
new Uri("s3://" + primaryBucket + "/" + bucketPath + "/"),
amazonAccessKeyID,
Expand Down

0 comments on commit 4bc43fc

Please sign in to comment.