Skip to content

Commit

Permalink
Add disableHTTPS and usePathStyle s3v2.Options as query param
Browse files Browse the repository at this point in the history
This ensures that we can use blob/blob with s3 compatible storage like minio.
Pass use_path_style=true to force PathStyle for s3.
Pass disable_https=true to disable tls for endpoint.
  • Loading branch information
khrm committed Oct 9, 2024
1 parent e5b1bc6 commit 0e94b90
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 6 deletions.
38 changes: 32 additions & 6 deletions blob/s3blob/s3blob.go
Original file line number Diff line number Diff line change
Expand Up @@ -150,9 +150,11 @@ type URLOpener struct {
}

const (
sseTypeParamKey = "ssetype"
kmsKeyIdParamKey = "kmskeyid"
accelerateParamKey = "accelerate"
sseTypeParamKey = "ssetype"
kmsKeyIdParamKey = "kmskeyid"
accelerateParamKey = "accelerate"
usePathStyleParamkey = "use_path_style"
disableHTTPSParamKey = "disable_https"
)

func toServerSideEncryptionType(value string) (typesv2.ServerSideEncryption, error) {
Expand Down Expand Up @@ -195,13 +197,37 @@ func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket
}

if o.UseV2 {
opts := []func(*s3v2.Options){
func(o *s3v2.Options) {
o.UseAccelerate = accelerate
},
}
if disableHTTPSParam := q.Get(disableHTTPSParamKey); disableHTTPSParam != "" {
q.Del(disableHTTPSParamKey)
value, err := strconv.ParseBool(disableHTTPSParam)
if err != nil {
return nil, fmt.Errorf("invalid value for %q: %v", disableHTTPSParamKey, err)

Check warning on line 209 in blob/s3blob/s3blob.go

View check run for this annotation

Codecov / codecov/patch

blob/s3blob/s3blob.go#L209

Added line #L209 was not covered by tests
}
opts = append(opts, func(o *s3v2.Options) {
o.EndpointOptions.DisableHTTPS = value
})
}
if usePathStyleParam := q.Get(usePathStyleParamkey); usePathStyleParam != "" {
q.Del(usePathStyleParamkey)
value, err := strconv.ParseBool(usePathStyleParam)
if err != nil {
return nil, fmt.Errorf("invalid value for %q: %v", usePathStyleParamkey, err)

Check warning on line 219 in blob/s3blob/s3blob.go

View check run for this annotation

Codecov / codecov/patch

blob/s3blob/s3blob.go#L219

Added line #L219 was not covered by tests
}
opts = append(opts, func(o *s3v2.Options) {
o.UsePathStyle = value
})
}

cfg, err := gcaws.V2ConfigFromURLParams(ctx, q)
if err != nil {
return nil, fmt.Errorf("open bucket %v: %v", u, err)
}
clientV2 := s3v2.NewFromConfig(cfg, func(o *s3v2.Options) {
o.UseAccelerate = accelerate
})
clientV2 := s3v2.NewFromConfig(cfg, opts...)

return OpenBucketV2(ctx, clientV2, u.Host, &o.Options)
}
Expand Down
8 changes: 8 additions & 0 deletions blob/s3blob/s3blob_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -482,6 +482,10 @@ func TestOpenBucketFromURL(t *testing.T) {
{"s3://mybucket?fips=true", false},
// OK, use S3 Transfer accleration and dual stack endpoints (v1)
{"s3://mybucket?awssdk=v1&accelerate=true&dualstack=true", false},
// OK, use use_path_style
{"s3://mybucket?use_path_style=true", false},
// OK, use disable_https
{"s3://mybucket?disable_https=true", false},
// OK, use FIPS endpoints (v1)
{"s3://mybucket?awssdk=v1&fips=true", false},
// Invalid accelerate (v1)
Expand All @@ -500,6 +504,10 @@ func TestOpenBucketFromURL(t *testing.T) {
{"s3://mybucket?ssetype=aws:notkmsoraes&kmskeyid=arn:aws:us-east-1:12345:key/1-a-2-b", true},
// Invalid parameter together with a valid one.
{"s3://mybucket?profile=main&param=value", true},
// Invalid use_path_style (v1)
{"s3://mybucket?awssdk=v1&usePathStyle=bad", true},
// Invalid disable_https (v2)
{"s3://mybucket?usePathStyle=bad", true},
// Invalid parameter.
{"s3://mybucket?param=value", true},
}
Expand Down

0 comments on commit 0e94b90

Please sign in to comment.