From 0863c808cf835a90461f2b24e9ced5683b5b8ee3 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Tue, 5 Nov 2024 19:15:59 -0800 Subject: [PATCH] update thanos objstore library to pull object duration metric fix Signed-off-by: Ben Ye --- go.mod | 2 +- go.sum | 4 +- .../thanos-io/objstore/CHANGELOG.md | 3 ++ vendor/github.com/thanos-io/objstore/inmem.go | 32 +++++++++++--- .../github.com/thanos-io/objstore/objstore.go | 18 +++++++- .../objstore/providers/azure/azure.go | 19 +++++---- .../objstore/providers/azure/helpers.go | 5 ++- .../providers/filesystem/filesystem.go | 29 ++++++++++--- .../thanos-io/objstore/providers/gcs/gcs.go | 42 ++++++++++++++----- .../thanos-io/objstore/providers/s3/s3.go | 25 +++++++---- .../objstore/providers/swift/swift.go | 26 +++++++----- .../github.com/thanos-io/objstore/testing.go | 20 +++++++++ vendor/modules.txt | 2 +- 13 files changed, 174 insertions(+), 53 deletions(-) diff --git a/go.mod b/go.mod index 2939a72006..4ced7db52d 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/sony/gobreaker v1.0.0 github.com/spf13/afero v1.11.0 github.com/stretchr/testify v1.9.0 - github.com/thanos-io/objstore v0.0.0-20241010161353-f90c89a0ef90 + github.com/thanos-io/objstore v0.0.0-20241028150459-cfdd0e50390d github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31 github.com/thanos-io/thanos v0.35.2-0.20241029125830-62038110b1bc github.com/uber/jaeger-client-go v2.30.0+incompatible diff --git a/go.sum b/go.sum index b70e10beb6..aa1f816ce4 100644 --- a/go.sum +++ b/go.sum @@ -1684,8 +1684,8 @@ github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4 github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1Zsv7OAU9iQhZwigp50Yl38W10g/vd5NC8Rdk1Jzng= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM= -github.com/thanos-io/objstore v0.0.0-20241010161353-f90c89a0ef90 h1:+gDIM0kLg4mpwU4RjvG09KiZsPrCy9EBHf8J/uJ+Ve0= -github.com/thanos-io/objstore v0.0.0-20241010161353-f90c89a0ef90/go.mod h1:/ZMUxFcp/nT6oYV5WslH9k07NU/+86+aibgZRmMMr/4= +github.com/thanos-io/objstore v0.0.0-20241028150459-cfdd0e50390d h1:k+SLTP1mjNqXxsCiq4UYeKCe07le0ieffyuHm/YfmH8= +github.com/thanos-io/objstore v0.0.0-20241028150459-cfdd0e50390d/go.mod h1:/ZMUxFcp/nT6oYV5WslH9k07NU/+86+aibgZRmMMr/4= github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31 h1:xPaP58g+3EPohdw4cv+6jv5+LcX6LynhHvQcYwTAMxQ= github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31/go.mod h1:wx0JlRZtsB2S10JYUgeg5GqLfMxw31SzArP+28yyE00= github.com/thanos-io/thanos v0.35.2-0.20241029125830-62038110b1bc h1:AoIDFFb3xjED+mmf5g6bnULDgNJWeXtKyjAQ1CKtqfk= diff --git a/vendor/github.com/thanos-io/objstore/CHANGELOG.md b/vendor/github.com/thanos-io/objstore/CHANGELOG.md index 12375cbf68..d2b1aaabda 100644 --- a/vendor/github.com/thanos-io/objstore/CHANGELOG.md +++ b/vendor/github.com/thanos-io/objstore/CHANGELOG.md @@ -10,8 +10,10 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ## Unreleased - [#38](https://github.com/thanos-io/objstore/pull/38) GCS: Upgrade cloud.google.com/go/storage version to `v1.43.0`. +- [#145](https://github.com/thanos-io/objstore/pull/145) Include content length in the response of Get and GetRange. ### Fixed +- [#153](https://github.com/thanos-io/objstore/pull/153) Metrics: Fix `objstore_bucket_operation_duration_seconds_*` for `get` and `get_range` operations. - [#117](https://github.com/thanos-io/objstore/pull/117) Metrics: Fix `objstore_bucket_operation_failures_total` incorrectly incremented if context is cancelled while reading object contents. - [#115](https://github.com/thanos-io/objstore/pull/115) GCS: Fix creation of bucket with GRPC connections. Also update storage client to `v1.40.0`. - [#102](https://github.com/thanos-io/objstore/pull/102) Azure: bump azblob sdk to get concurrency fixes. @@ -51,6 +53,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#116](https://github.com/thanos-io/objstore/pull/116) Azure: Add new storage_create_container configuration property - [#128](https://github.com/thanos-io/objstore/pull/128) GCS: Add support for `ChunkSize` for writer. - [#130](https://github.com/thanos-io/objstore/pull/130) feat: Decouple creating bucket metrics from instrumenting the bucket +- [#150](https://github.com/thanos-io/objstore/pull/150) Add support for roundtripper wrapper. ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. diff --git a/vendor/github.com/thanos-io/objstore/inmem.go b/vendor/github.com/thanos-io/objstore/inmem.go index 3f6f35e94e..ed256c9cd9 100644 --- a/vendor/github.com/thanos-io/objstore/inmem.go +++ b/vendor/github.com/thanos-io/objstore/inmem.go @@ -119,7 +119,12 @@ func (b *InMemBucket) Get(_ context.Context, name string) (io.ReadCloser, error) return nil, errNotFound } - return io.NopCloser(bytes.NewReader(file)), nil + return ObjectSizerReadCloser{ + ReadCloser: io.NopCloser(bytes.NewReader(file)), + Size: func() (int64, error) { + return int64(len(file)), nil + }, + }, nil } // GetRange returns a new range reader for the given object name and range. @@ -136,15 +141,27 @@ func (b *InMemBucket) GetRange(_ context.Context, name string, off, length int64 } if int64(len(file)) < off { - return io.NopCloser(bytes.NewReader(nil)), nil + return ObjectSizerReadCloser{ + ReadCloser: io.NopCloser(bytes.NewReader(nil)), + Size: func() (int64, error) { return 0, nil }, + }, nil } if length == -1 { - return io.NopCloser(bytes.NewReader(file[off:])), nil + return ObjectSizerReadCloser{ + ReadCloser: io.NopCloser(bytes.NewReader(file[off:])), + Size: func() (int64, error) { + return int64(len(file[off:])), nil + }, + }, nil } if length <= 0 { - return io.NopCloser(bytes.NewReader(nil)), errors.New("length cannot be smaller or equal 0") + // wrap with ObjectSizerReadCloser to return 0 size. + return ObjectSizerReadCloser{ + ReadCloser: io.NopCloser(bytes.NewReader(nil)), + Size: func() (int64, error) { return 0, nil }, + }, errors.New("length cannot be smaller or equal 0") } if int64(len(file)) <= off+length { @@ -152,7 +169,12 @@ func (b *InMemBucket) GetRange(_ context.Context, name string, off, length int64 length = int64(len(file)) - off } - return io.NopCloser(bytes.NewReader(file[off : off+length])), nil + return ObjectSizerReadCloser{ + ReadCloser: io.NopCloser(bytes.NewReader(file[off : off+length])), + Size: func() (int64, error) { + return length, nil + }, + }, nil } // Exists checks if the given directory exists in memory. diff --git a/vendor/github.com/thanos-io/objstore/objstore.go b/vendor/github.com/thanos-io/objstore/objstore.go index bfee75037a..62f1c655db 100644 --- a/vendor/github.com/thanos-io/objstore/objstore.go +++ b/vendor/github.com/thanos-io/objstore/objstore.go @@ -573,7 +573,7 @@ func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, err if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.metrics.opsFailures.WithLabelValues(op).Inc() } - b.metrics.opsDuration.WithLabelValues(op).Observe(float64(time.Since(start))) + b.metrics.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) return nil, err } return newTimingReader( @@ -600,7 +600,7 @@ func (b *metricBucket) GetRange(ctx context.Context, name string, off, length in if !b.metrics.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.metrics.opsFailures.WithLabelValues(op).Inc() } - b.metrics.opsDuration.WithLabelValues(op).Observe(float64(time.Since(start))) + b.metrics.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) return nil, err } return newTimingReader( @@ -829,3 +829,17 @@ func (t *timingReaderWriterTo) WriteTo(w io.Writer) (n int64, err error) { t.timingReader.updateMetrics(int(n), err) return n, err } + +type ObjectSizerReadCloser struct { + io.ReadCloser + Size func() (int64, error) +} + +// ObjectSize implement ObjectSizer. +func (o ObjectSizerReadCloser) ObjectSize() (int64, error) { + if o.Size == nil { + return 0, errors.New("unknown size") + } + + return o.Size() +} diff --git a/vendor/github.com/thanos-io/objstore/providers/azure/azure.go b/vendor/github.com/thanos-io/objstore/providers/azure/azure.go index c2f3adb517..e125ca3511 100644 --- a/vendor/github.com/thanos-io/objstore/providers/azure/azure.go +++ b/vendor/github.com/thanos-io/objstore/providers/azure/azure.go @@ -146,7 +146,7 @@ type Bucket struct { } // NewBucket returns a new Bucket using the provided Azure config. -func NewBucket(logger log.Logger, azureConfig []byte, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucket(logger log.Logger, azureConfig []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { level.Debug(logger).Log("msg", "creating new Azure bucket connection", "component", component) conf, err := parseConfig(azureConfig) if err != nil { @@ -155,19 +155,16 @@ func NewBucket(logger log.Logger, azureConfig []byte, component string, rt http. if conf.MSIResource != "" { level.Warn(logger).Log("msg", "The field msi_resource has been deprecated and should no longer be set") } - return NewBucketWithConfig(logger, conf, component, rt) + return NewBucketWithConfig(logger, conf, component, wrapRoundtripper) } // NewBucketWithConfig returns a new Bucket using the provided Azure config struct. -func NewBucketWithConfig(logger log.Logger, conf Config, component string, rt http.RoundTripper) (*Bucket, error) { - if rt != nil { - conf.HTTPConfig.Transport = rt - } +func NewBucketWithConfig(logger log.Logger, conf Config, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { if err := conf.validate(); err != nil { return nil, err } - containerClient, err := getContainerClient(conf) + containerClient, err := getContainerClient(conf, wrapRoundtripper) if err != nil { return nil, err } @@ -273,7 +270,13 @@ func (b *Bucket) getBlobReader(ctx context.Context, name string, httpRange blob. return nil, errors.Wrapf(err, "cannot download blob, address: %s", blobClient.URL()) } retryOpts := azblob.RetryReaderOptions{MaxRetries: int32(b.readerMaxRetries)} - return resp.NewRetryReader(ctx, &retryOpts), nil + + return objstore.ObjectSizerReadCloser{ + ReadCloser: resp.NewRetryReader(ctx, &retryOpts), + Size: func() (int64, error) { + return *resp.ContentLength, nil + }, + }, nil } // Get returns a reader for the given object name. diff --git a/vendor/github.com/thanos-io/objstore/providers/azure/helpers.go b/vendor/github.com/thanos-io/objstore/providers/azure/helpers.go index 2915fbbbc8..deb86d03d0 100644 --- a/vendor/github.com/thanos-io/objstore/providers/azure/helpers.go +++ b/vendor/github.com/thanos-io/objstore/providers/azure/helpers.go @@ -19,7 +19,7 @@ import ( // DirDelim is the delimiter used to model a directory structure in an object store bucket. const DirDelim = "/" -func getContainerClient(conf Config) (*container.Client, error) { +func getContainerClient(conf Config, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*container.Client, error) { var rt http.RoundTripper rt, err := exthttp.DefaultTransport(conf.HTTPConfig) if err != nil { @@ -28,6 +28,9 @@ func getContainerClient(conf Config) (*container.Client, error) { if conf.HTTPConfig.Transport != nil { rt = conf.HTTPConfig.Transport } + if wrapRoundtripper != nil { + rt = wrapRoundtripper(rt) + } opt := &container.ClientOptions{ ClientOptions: azcore.ClientOptions{ Retry: policy.RetryOptions{ diff --git a/vendor/github.com/thanos-io/objstore/providers/filesystem/filesystem.go b/vendor/github.com/thanos-io/objstore/providers/filesystem/filesystem.go index 21c7048505..2ed42ee8b6 100644 --- a/vendor/github.com/thanos-io/objstore/providers/filesystem/filesystem.go +++ b/vendor/github.com/thanos-io/objstore/providers/filesystem/filesystem.go @@ -150,8 +150,12 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( return nil, errors.New("object name is empty") } - file := filepath.Join(b.rootDir, name) - if _, err := os.Stat(file); err != nil { + var ( + file = filepath.Join(b.rootDir, name) + stat os.FileInfo + err error + ) + if stat, err = os.Stat(file); err != nil { return nil, errors.Wrapf(err, "stat %s", file) } @@ -160,18 +164,33 @@ func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) ( return nil, err } + var newOffset int64 if off > 0 { - _, err := f.Seek(off, 0) + newOffset, err = f.Seek(off, 0) if err != nil { return nil, errors.Wrapf(err, "seek %v", off) } } + size := stat.Size() - newOffset if length == -1 { - return f, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: f, + Size: func() (int64, error) { + return size, nil + }, + }, nil } - return &rangeReaderCloser{Reader: io.LimitReader(f, length), f: f}, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: &rangeReaderCloser{ + Reader: io.LimitReader(f, length), + f: f, + }, + Size: func() (int64, error) { + return min(length, size), nil + }, + }, nil } // Exists checks if the given directory exists in memory. diff --git a/vendor/github.com/thanos-io/objstore/providers/gcs/gcs.go b/vendor/github.com/thanos-io/objstore/providers/gcs/gcs.go index b5dae20055..efb208e60e 100644 --- a/vendor/github.com/thanos-io/objstore/providers/gcs/gcs.go +++ b/vendor/github.com/thanos-io/objstore/providers/gcs/gcs.go @@ -77,22 +77,20 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket against the given bucket handle. -func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, err } - return NewBucketWithConfig(ctx, logger, config, component, rt) + return NewBucketWithConfig(ctx, logger, config, component, wrapRoundtripper) } // NewBucketWithConfig returns a new Bucket with gcs Config struct. -func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { if gc.Bucket == "" { return nil, errors.New("missing Google Cloud Storage bucket name for stored blocks") } - if rt != nil { - gc.HTTPConfig.Transport = rt - } + var opts []option.ClientOption // If ServiceAccount is provided, use them in GCS client, otherwise fallback to Google default logic. @@ -112,7 +110,7 @@ func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, comp if !gc.UseGRPC { var err error - opts, err = appendHttpOptions(gc, opts) + opts, err = appendHttpOptions(gc, opts, wrapRoundtripper) if err != nil { return nil, err } @@ -121,7 +119,7 @@ func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, comp return newBucket(ctx, logger, gc, opts) } -func appendHttpOptions(gc Config, opts []option.ClientOption) ([]option.ClientOption, error) { +func appendHttpOptions(gc Config, opts []option.ClientOption, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) ([]option.ClientOption, error) { // Check if a roundtripper has been set in the config // otherwise build the default transport. var rt http.RoundTripper @@ -132,6 +130,9 @@ func appendHttpOptions(gc Config, opts []option.ClientOption) ([]option.ClientOp if gc.HTTPConfig.Transport != nil { rt = gc.HTTPConfig.Transport } + if wrapRoundtripper != nil { + rt = wrapRoundtripper(rt) + } // GCS uses some defaults when "options.WithHTTPClient" is not used that are important when we call // htransport.NewTransport namely the scopes that are then used for OAth authentication. So to build our own @@ -226,12 +227,33 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt // Get returns a reader for the given object name. func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { - return b.bkt.Object(name).NewReader(ctx) + r, err := b.bkt.Object(name).NewReader(ctx) + if err != nil { + return r, err + } + + return objstore.ObjectSizerReadCloser{ + ReadCloser: r, + Size: func() (int64, error) { + return r.Attrs.Size, nil + }, + }, nil } // GetRange returns a new range reader for the given object name and range. func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { - return b.bkt.Object(name).NewRangeReader(ctx, off, length) + r, err := b.bkt.Object(name).NewRangeReader(ctx, off, length) + if err != nil { + return r, err + } + + sz := r.Remain() + return objstore.ObjectSizerReadCloser{ + ReadCloser: r, + Size: func() (int64, error) { + return sz, nil + }, + }, nil } // Attributes returns information about the specified object. diff --git a/vendor/github.com/thanos-io/objstore/providers/s3/s3.go b/vendor/github.com/thanos-io/objstore/providers/s3/s3.go index 2f0447a796..8e5b8b5640 100644 --- a/vendor/github.com/thanos-io/objstore/providers/s3/s3.go +++ b/vendor/github.com/thanos-io/objstore/providers/s3/s3.go @@ -176,13 +176,13 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket using the provided s3 config values. -func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, err } - return NewBucketWithConfig(logger, config, component, rt) + return NewBucketWithConfig(logger, config, component, wrapRoundtripper) } type overrideSignerType struct { @@ -202,7 +202,7 @@ func (s *overrideSignerType) Retrieve() (credentials.Value, error) { } // NewBucketWithConfig returns a new Bucket using the provided s3 config values. -func NewBucketWithConfig(logger log.Logger, config Config, component string, rt http.RoundTripper) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error) { var chain []credentials.Provider // TODO(bwplotka): Don't do flags as they won't scale, use actual params like v2, v4 instead @@ -242,9 +242,7 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, rt }), } } - if rt != nil { - config.HTTPConfig.Transport = rt - } + // Check if a roundtripper has been set in the config // otherwise build the default transport. var tpt http.RoundTripper @@ -255,6 +253,9 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, rt if config.HTTPConfig.Transport != nil { tpt = config.HTTPConfig.Transport } + if wrapRoundtripper != nil { + tpt = wrapRoundtripper(tpt) + } client, err := minio.New(config.Endpoint, &minio.Options{ Creds: credentials.NewChainCredentials(chain), @@ -452,7 +453,17 @@ func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) ( return nil, err } - return r, nil + return objstore.ObjectSizerReadCloser{ + ReadCloser: r, + Size: func() (int64, error) { + stat, err := r.Stat() + if err != nil { + return 0, err + } + + return stat.Size, nil + }, + }, nil } // Get returns a reader for the given object name. diff --git a/vendor/github.com/thanos-io/objstore/providers/swift/swift.go b/vendor/github.com/thanos-io/objstore/providers/swift/swift.go index 682c494cab..e872728e4d 100644 --- a/vendor/github.com/thanos-io/objstore/providers/swift/swift.go +++ b/vendor/github.com/thanos-io/objstore/providers/swift/swift.go @@ -154,12 +154,12 @@ type Container struct { segmentsContainer string } -func NewContainer(logger log.Logger, conf []byte, rt http.RoundTripper) (*Container, error) { +func NewContainer(logger log.Logger, conf []byte, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Container, error) { sc, err := parseConfig(conf) if err != nil { return nil, errors.Wrap(err, "parse config") } - return NewContainerFromConfig(logger, sc, false, rt) + return NewContainerFromConfig(logger, sc, false, wrapRoundtripper) } func ensureContainer(connection *swift.Connection, name string, createIfNotExist bool) error { @@ -178,22 +178,22 @@ func ensureContainer(connection *swift.Connection, name string, createIfNotExist return nil } -func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool, rt http.RoundTripper) (*Container, error) { - if rt != nil { - sc.HTTPConfig.Transport = rt - } +func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Container, error) { // Check if a roundtripper has been set in the config // otherwise build the default transport. - var tpt http.RoundTripper - tpt, err := exthttp.DefaultTransport(sc.HTTPConfig) + var rt http.RoundTripper + rt, err := exthttp.DefaultTransport(sc.HTTPConfig) if err != nil { return nil, err } if sc.HTTPConfig.Transport != nil { - tpt = sc.HTTPConfig.Transport + rt = sc.HTTPConfig.Transport + } + if wrapRoundtripper != nil { + rt = wrapRoundtripper(rt) } - connection := connectionFromConfig(sc, tpt) + connection := connectionFromConfig(sc, rt) if err := connection.Authenticate(); err != nil { return nil, errors.Wrap(err, "authentication") } @@ -262,7 +262,11 @@ func (c *Container) get(name string, headers swift.Headers, checkHash bool) (io. if err != nil { return nil, errors.Wrap(err, "open object") } - return file, err + + return objstore.ObjectSizerReadCloser{ + ReadCloser: file, + Size: file.Length, + }, nil } // Get returns a reader for the given object name. diff --git a/vendor/github.com/thanos-io/objstore/testing.go b/vendor/github.com/thanos-io/objstore/testing.go index b8e3744cb8..28cbd65889 100644 --- a/vendor/github.com/thanos-io/objstore/testing.go +++ b/vendor/github.com/thanos-io/objstore/testing.go @@ -106,6 +106,11 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { rc1, err := bkt.Get(ctx, "id1/obj_1.some") testutil.Ok(t, err) defer func() { testutil.Ok(t, rc1.Close()) }() + + sz, err := TryToGetSize(rc1) + testutil.Ok(t, err) + testutil.Equals(t, int64(11), sz, "expected size to be equal to 11") + content, err := io.ReadAll(rc1) testutil.Ok(t, err) testutil.Equals(t, "@test-data@", string(content)) @@ -118,6 +123,11 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { rc2, err := bkt.GetRange(ctx, "id1/obj_1.some", 1, 3) testutil.Ok(t, err) defer func() { testutil.Ok(t, rc2.Close()) }() + + sz, err = TryToGetSize(rc2) + testutil.Ok(t, err) + testutil.Equals(t, int64(3), sz, "expected size to be equal to 3") + content, err = io.ReadAll(rc2) testutil.Ok(t, err) testutil.Equals(t, "tes", string(content)) @@ -126,6 +136,11 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { rcUnspecifiedLen, err := bkt.GetRange(ctx, "id1/obj_1.some", 1, -1) testutil.Ok(t, err) defer func() { testutil.Ok(t, rcUnspecifiedLen.Close()) }() + + sz, err = TryToGetSize(rcUnspecifiedLen) + testutil.Ok(t, err) + testutil.Equals(t, int64(10), sz, "expected size to be equal to 10") + content, err = io.ReadAll(rcUnspecifiedLen) testutil.Ok(t, err) testutil.Equals(t, "test-data@", string(content)) @@ -141,6 +156,11 @@ func AcceptanceTest(t *testing.T, bkt Bucket) { rcLength, err := bkt.GetRange(ctx, "id1/obj_1.some", 3, 9999) testutil.Ok(t, err) defer func() { testutil.Ok(t, rcLength.Close()) }() + + sz, err = TryToGetSize(rcLength) + testutil.Ok(t, err) + testutil.Equals(t, int64(8), sz, "expected size to be equal to 8") + content, err = io.ReadAll(rcLength) testutil.Ok(t, err) testutil.Equals(t, "st-data@", string(content)) diff --git a/vendor/modules.txt b/vendor/modules.txt index 72a54ad4ff..52516b3428 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -949,7 +949,7 @@ github.com/stretchr/objx github.com/stretchr/testify/assert github.com/stretchr/testify/mock github.com/stretchr/testify/require -# github.com/thanos-io/objstore v0.0.0-20241010161353-f90c89a0ef90 +# github.com/thanos-io/objstore v0.0.0-20241028150459-cfdd0e50390d ## explicit; go 1.21 github.com/thanos-io/objstore github.com/thanos-io/objstore/exthttp