Update README file and refactor code for improved readability

Signed-off-by: Kushal Shukla <[email protected]>

Author: kushalShukla-web

tools/object-storage/Dockerfile tools/blockSyncer/Dockerfile

@@ -1,6 +1,6 @@
 FROM quay.io/prometheus/busybox:latest
 LABEL maintainer="The Prometheus Authors <[email protected]>"
 
-COPY object-storage /bin/object-storage
+COPY blockSyncer /bin/blocksyncer
 
-ENTRYPOINT ["/bin/object-storage"]
+ENTRYPOINT ["/bin/blocksyncer"]

tools/blockSyncer/Readme.md

@@ -0,0 +1,53 @@
+
+# blockSynch - TSDB Data Synchronization Tool
+
+
+The `block-sync` command is a CLI tool designed to synchronize TSDB data with an object storage system. 
+
+## Table of Contents
+
+1. [Upload](#upload)
+2. [Download](#download)
+
+## Command Flags
+
+- ``` -h , --help```:Displays context-sensitive help 
+- ``` - tsdb-path```: Path for The TSDB data in prometheus
+- ```- objstore.config-file```: Path for The Config file
+- ```- Key```: Path for the Key where to store block data , i.e a Directory. 
+
+## Upload
+
+The `upload` command allows you to upload TSDB data from a specified path to an object storage bucket. This command is essential for backing up your TSDB data or migrating it to an object storage solution for future use.
+
+### Usage
+
+```bash
+./blockSynch upload --tsdb-path=<path-to-tsdb> --objstore.config-file=<path-to-config> --key=<object-key>
+
+
+```
+## Download
+
+The `download` command allows you to retrieve TSDB data from an object storage bucket to a specified local path. This command is essential for restoring your TSDB data or retrieving it for local analysis and processing.
+
+### Usage
+
+```bash
+./blockSynch download --tsdb-path=<path-to-tsdb> --objstore.config-file=<path-to-config> --key=<object-key>
+```
+## Config File
+
+The configuration file is essential for connecting to your object storage solution. Below are basic templates for different object storage systems.
+
+```yaml
+type: s3
+config:
+  bucket: your-bucket-name
+  endpoint: https://your-minio-endpoint.com
+  access_key: your-access-key
+  secret_key: your-secret-key
+  insecure: false  # Set to true if using HTTP instead of HTTPS
+```
+For more follow this link [Storage.md](https://thanos.io/tip/thanos/storage.md/)
+

tools/blockSyncer/obj.go

@@ -0,0 +1,50 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	"gopkg.in/alecthomas/kingpin.v2"
+)
+
+func main() {
+	app := kingpin.New(filepath.Base(os.Args[0]), "Tool for storing TSDB data to object storage")
+	app.HelpFlag.Short('h')
+
+	var tsdbPath, objectConfig, objectKey string
+	var s *Store
+
+	objstore := app.Command("block-sync", `Using an object storage to store the data`)
+	objstore.Flag("tsdb-path", "Path for The TSDB data in prometheus").Required().StringVar(&tsdbPath)
+	objstore.Flag("objstore.config-file", "Path for The Config file").Required().StringVar(&objectConfig)
+	objstore.Flag("key", "Path for the Key where to store block data").Required().StringVar(&objectKey)
+
+	objstore.Action(func(c *kingpin.ParseContext) error {
+		s = newstore(tsdbPath, objectConfig, objectKey)
+		return nil
+	})
+
+	uploadCmd := objstore.Command("upload", "Uploading data")
+	uploadCmd.Action(func(c *kingpin.ParseContext) error {
+		return s.upload(c)
+	})
+
+	downloadCmd := objstore.Command("download", "Downloading data")
+	downloadCmd.Action(func(c *kingpin.ParseContext) error {
+		return s.download(c)
+	})
+	kingpin.MustParse(app.Parse(os.Args[1:]))
+}

tools/blockSyncer/upload_download.go

@@ -0,0 +1,90 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+
+	"github.com/go-kit/log"
+	"github.com/thanos-io/objstore"
+	"github.com/thanos-io/objstore/client"
+	"gopkg.in/alecthomas/kingpin.v2"
+)
+
+type Store struct {
+	Bucket       objstore.Bucket
+	TsdbPath     string
+	ObjectKey    string
+	ObjectConfig string
+}
+
+func newstore(tsdbPath, objectConfig, objectKey string) *Store {
+	logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
+	configBytes, err := os.ReadFile(objectConfig)
+	if err != nil {
+		logger.Error("failed to read config file", "error", err)
+		return nil
+	}
+
+	bucket, err := client.NewBucket(log.NewNopLogger(), configBytes, "block-sync")
+	if err != nil {
+		logger.Error("Failed to create bucket existence", "error", err)
+		return nil
+	}
+
+	exists, err := bucket.Exists(context.Background(), objectKey)
+	if err != nil {
+		logger.Error("Failed to create new bucket", "error", err)
+		return nil
+	}
+	logger.Info("Bucket existence check", "bucket_name", objectKey, "exists", exists)
+
+	return &Store{
+		Bucket:       bucket,
+		TsdbPath:     tsdbPath,
+		ObjectConfig: objectConfig,
+		ObjectKey:    objectKey,
+	}
+}
+
+func (c *Store) upload(*kingpin.ParseContext) error {
+
+	logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
+
+	err := objstore.UploadDir(context.Background(), log.NewNopLogger(), c.Bucket, c.TsdbPath, c.ObjectKey)
+	if err != nil {
+		logger.Error("Failed to upload directory", "path", c.TsdbPath, "error", err)
+		return fmt.Errorf("failed to upload directory from path %s to bucket: %v", c.TsdbPath, err)
+	}
+
+	logger.Info("Successfully uploaded directory", "path", c.TsdbPath, "bucket", c.Bucket.Name())
+	return nil
+}
+
+func (c *Store) download(*kingpin.ParseContext) error {
+
+	logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
+
+	err := objstore.DownloadDir(context.Background(), log.NewNopLogger(), c.Bucket, "dir/", c.ObjectKey, c.TsdbPath)
+	if err != nil {
+		logger.Error("Failed to download directory", "path", c.TsdbPath, "error", err)
+		return fmt.Errorf("failed to download directory from path %s to bucket: %v", c.TsdbPath, err)
+	}
+
+	logger.Info("Successfully downloaded directory", "path", c.TsdbPath, "bucket", c.Bucket.Name())
+	return nil
+}