Add Checksum to PutObject (#2002)

Requires TrailingHeaders to be enabled on the client.

Mint test updated.

Verified against AWS:

```
{"time":"2024-09-19T12:54:20.5068834+02:00","level":"INFO","name":"minio-go: testPutObjectWithChecksums","duration":3504,"function":"PutObject(bucketName, objectName, reader,size, opts)","args":{"bucketName":"minio-go-test-qgn9l4slibugev06","checksum":"SHA256","objectName":"9oady6pvtp30mjatsc2141txeex5vc","opts":"minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}"},"status":"PASS"}
{"time":"2024-09-19T12:54:24.1432974+02:00","level":"INFO","name":"minio-go: testPutObjectWithTrailingChecksums","duration":2742,"function":"PutObject(bucketName, objectName, reader,size, opts)","args":{"bucketName":"minio-go-test-wxrdafbi5xofe9pq","checksum":"SHA256","objectName":"ih64zss0yjp4g2v5xt1atg6y9ouwwk","opts":"minio.PutObjectOptions{UserMetadata: metadata, Progress: progress, TrailChecksum: xxx}"},"status":"PASS"}
{"time":"2024-09-19T12:58:44.9830104+02:00","level":"INFO","name":"minio-go: testPutMultipartObjectWithChecksums","duration":259866,"function":"PutObject(bucketName, objectName, reader,size, opts)","args":{"bucketName":"minio-go-test-40wbkvvk55n5whld","checksum":"SHA256","objectName":"c4kvdfsdcvzk94j1oudy9th6es6cvy","opts":"minio.PutObjectOptions{UserMetadata: metadata, Progress: progress Checksum: false}"},"status":"PASS"}
{"time":"2024-09-19T12:59:22.7002636+02:00","level":"INFO","name":"minio-go: testPutMultipartObjectWithChecksums","duration":36731,"function":"PutObject(bucketName, objectName, reader,size, opts)","args":{"bucketName":"minio-go-test-mwfrpr43lfe6sc6o","checksum":"SHA256","objectName":"1m3fzred3ldar5gl253gspan0ltzma","opts":"minio.PutObjectOptions{UserMetadata: metadata, Progress: progress Checksum: true}"},"status":"PASS"}
```

- Fix up testGetObjectAttributes using non-explicit checksum
- Fix TLS transports, with SKIP_CERT_VALIDATION

Author: klauspost

api-put-object-streaming.go

@@ -108,7 +108,9 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
 	if err != nil {
 		return UploadInfo{}, err
 	}
-
+	if opts.Checksum.IsSet() {
+		opts.AutoChecksum = opts.Checksum
+	}
 	withChecksum := c.trailingHeaderSupport
 	if withChecksum {
 		if opts.UserMetadata == nil {
@@ -304,6 +306,11 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
 		return UploadInfo{}, err
 	}
 
+	if opts.Checksum.IsSet() {
+		opts.AutoChecksum = opts.Checksum
+		opts.SendContentMd5 = false
+	}
+
 	if !opts.SendContentMd5 {
 		if opts.UserMetadata == nil {
 			opts.UserMetadata = make(map[string]string, 1)
@@ -463,7 +470,10 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam
 	if err = s3utils.CheckValidObjectName(objectName); err != nil {
 		return UploadInfo{}, err
 	}
-
+	if opts.Checksum.IsSet() {
+		opts.SendContentMd5 = false
+		opts.AutoChecksum = opts.Checksum
+	}
 	if !opts.SendContentMd5 {
 		if opts.UserMetadata == nil {
 			opts.UserMetadata = make(map[string]string, 1)
@@ -555,7 +565,7 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam
 		// Calculate md5sum.
 		customHeader := make(http.Header)
 		if !opts.SendContentMd5 {
-			// Add CRC32C instead.
+			// Add Checksum instead.
 			crc.Reset()
 			crc.Write(buf[:length])
 			cSum := crc.Sum(nil)
@@ -677,6 +687,9 @@ func (c *Client) putObject(ctx context.Context, bucketName, objectName string, r
 	if opts.SendContentMd5 && s3utils.IsGoogleEndpoint(*c.endpointURL) && size < 0 {
 		return UploadInfo{}, errInvalidArgument("MD5Sum cannot be calculated with size '-1'")
 	}
+	if opts.Checksum.IsSet() {
+		opts.SendContentMd5 = false
+	}
 
 	var readSeeker io.Seeker
 	if size > 0 {
@@ -746,17 +759,6 @@ func (c *Client) putObjectDo(ctx context.Context, bucketName, objectName string,
 	// Set headers.
 	customHeader := opts.Header()
 
-	// Add CRC when client supports it, MD5 is not set, not Google and we don't add SHA256 to chunks.
-	addCrc := c.trailingHeaderSupport && md5Base64 == "" && !s3utils.IsGoogleEndpoint(*c.endpointURL) && (opts.DisableContentSha256 || c.secure)
-
-	if addCrc {
-		// If user has added checksums, don't add them ourselves.
-		for k := range opts.UserMetadata {
-			if strings.HasPrefix(strings.ToLower(k), "x-amz-checksum-") {
-				addCrc = false
-			}
-		}
-	}
 	// Populate request metadata.
 	reqMetadata := requestMetadata{
 		bucketName:       bucketName,
@@ -768,10 +770,23 @@ func (c *Client) putObjectDo(ctx context.Context, bucketName, objectName string,
 		contentSHA256Hex: sha256Hex,
 		streamSha256:     !opts.DisableContentSha256,
 	}
-	if addCrc {
-		opts.AutoChecksum.SetDefault(ChecksumCRC32C)
-		reqMetadata.addCrc = &opts.AutoChecksum
+	// Add CRC when client supports it, MD5 is not set, not Google and we don't add SHA256 to chunks.
+	addCrc := c.trailingHeaderSupport && md5Base64 == "" && !s3utils.IsGoogleEndpoint(*c.endpointURL) && (opts.DisableContentSha256 || c.secure)
+	if opts.Checksum.IsSet() {
+		reqMetadata.addCrc = &opts.Checksum
+	} else if addCrc {
+		// If user has added checksums, don't add them ourselves.
+		for k := range opts.UserMetadata {
+			if strings.HasPrefix(strings.ToLower(k), "x-amz-checksum-") {
+				addCrc = false
+			}
+		}
+		if addCrc {
+			opts.AutoChecksum.SetDefault(ChecksumCRC32C)
+			reqMetadata.addCrc = &opts.AutoChecksum
+		}
 	}
+
 	if opts.Internal.SourceVersionID != "" {
 		if opts.Internal.SourceVersionID != nullVersionID {
 			if _, err := uuid.Parse(opts.Internal.SourceVersionID); err != nil {

api-put-object.go

@@ -94,6 +94,13 @@ type PutObjectOptions struct {
 	// If none is specified CRC32C is used, since it is generally the fastest.
 	AutoChecksum ChecksumType
 
+	// Checksum will force a checksum of the specific type.
+	// This requires that the client was created with "TrailingHeaders:true" option,
+	// and that the destination server supports it.
+	// Unavailable with V2 signatures & Google endpoints.
+	// This will disable content MD5 checksums if set.
+	Checksum ChecksumType
+
 	// ConcurrentStreamParts will create NumThreads buffers of PartSize bytes,
 	// fill them serially and upload them in parallel.
 	// This can be used for faster uploads on non-seekable or slow-to-seek input.
@@ -240,7 +247,7 @@ func (opts PutObjectOptions) Header() (header http.Header) {
 }
 
 // validate() checks if the UserMetadata map has standard headers or and raises an error if so.
-func (opts PutObjectOptions) validate() (err error) {
+func (opts PutObjectOptions) validate(c *Client) (err error) {
 	for k, v := range opts.UserMetadata {
 		if !httpguts.ValidHeaderFieldName(k) || isStandardHeader(k) || isSSEHeader(k) || isStorageClassHeader(k) || isMinioHeader(k) {
 			return errInvalidArgument(k + " unsupported user defined metadata name")
@@ -255,6 +262,17 @@ func (opts PutObjectOptions) validate() (err error) {
 	if opts.LegalHold != "" && !opts.LegalHold.IsValid() {
 		return errInvalidArgument(opts.LegalHold.String() + " unsupported legal-hold status")
 	}
+	if opts.Checksum.IsSet() {
+		switch {
+		case !c.trailingHeaderSupport:
+			return errInvalidArgument("Checksum requires Client with TrailingHeaders enabled")
+		case c.overrideSignerType.IsV2():
+			return errInvalidArgument("Checksum cannot be used with v2 signatures")
+		case s3utils.IsGoogleEndpoint(*c.endpointURL):
+			return errInvalidArgument("Checksum cannot be used with GCS endpoints")
+		}
+	}
+
 	return nil
 }
 
@@ -291,7 +309,7 @@ func (c *Client) PutObject(ctx context.Context, bucketName, objectName string, r
 		return UploadInfo{}, errors.New("object size must be provided with disable multipart upload")
 	}
 
-	err = opts.validate()
+	err = opts.validate(c)
 	if err != nil {
 		return UploadInfo{}, err
 	}
@@ -362,6 +380,10 @@ func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketNam
 		return UploadInfo{}, err
 	}
 
+	if opts.Checksum.IsSet() {
+		opts.SendContentMd5 = false
+		opts.AutoChecksum = opts.Checksum
+	}
 	if !opts.SendContentMd5 {
 		if opts.UserMetadata == nil {
 			opts.UserMetadata = make(map[string]string, 1)

api-put-object_test.go

@@ -61,7 +61,7 @@ func TestPutObjectOptionsValidate(t *testing.T) {
 	for i, testCase := range testCases {
 		err := PutObjectOptions{UserMetadata: map[string]string{
 			testCase.name: testCase.value,
-		}}.validate()
+		}}.validate(nil)
 		if testCase.shouldPass && err != nil {
 			t.Errorf("Test %d - output did not match with reference results, %s", i+1, err)
 		}

api-putobject-snowball.go

@@ -107,7 +107,7 @@ type readSeekCloser interface {
 // Total size should be < 5TB.
 // This function blocks until 'objs' is closed and the content has been uploaded.
 func (c Client) PutObjectsSnowball(ctx context.Context, bucketName string, opts SnowballOptions, objs <-chan SnowballObject) (err error) {
-	err = opts.Opts.validate()
+	err = opts.Opts.validate(&c)
 	if err != nil {
 		return err
 	}