1. Added key.yml

2. changed upload-download function to extract key
3. updated bash.sh file

Signed-off-by: Kushal Shukla <[email protected]>

Author: kushalShukla-web

prombench/manifests/prombench/benchmark/2a_prometheus-secret.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: s3-secret
+  namespace: prombench-{{ .PR_NUMBER }}  # Replace with your actual namespace
+type: Opaque
+stringData:
+  object-config.yml: |
+    type: S3
+    config:
+      bucket: {{ .BUCKET_KEY }}
+      endpoint: minio:9000  
+      access_key: {{ .ACCESS_KEY }}
+      secret_key: {{ .SECRET_KEY }}
+      insecure: true
+

prombench/manifests/prombench/benchmark/3b_prometheus-test_deployment.yaml

@@ -44,25 +44,34 @@ spec:
         - name: GITHUB_ORG
           value: "{{ .GITHUB_ORG }}"
         - name: GITHUB_REPO
-          value: "{{ .GITHUB_REPO }}"
+          value: "{{ .GITHUB_REPO }}"     
         volumeMounts:
         - name: prometheus-executable
           mountPath: /prometheus-builder
-        - name: config
-          mountPath: /config  
+        - name: key
+          mountPath: /config     
       - name: data-downloader
         image: kushalshukla/writer
-        imagePullPolicy: Always        
+        imagePullPolicy: Always
+        env:
+        - name: ACCESS_KEY
+          value: "{{ .ACCESS_KEY }}"
+        - name: SECRET_KEY
+          value: "{{ .SECRET_KEY }}"  
+        - name: BUCKET_KEY
+          value: "{{ .BUCKET_KEY }}"                
         volumeMounts:
         - name: instance-ssd
           mountPath: /data
-        - name: config
-          mountPath: /mnc                                   
+        - name: s3-config
+          mountPath: /config
+        - name: key
+          mountPath: /key 
         args: [
           "download",
         "--tsdb-path=/data", 
-        "--objstore.config-file=/mnc/objectconfig.yml",
-        "--key=gendata"
+        "--objstore.config-file=/config/object-config.yml",
+        "--key=/key/key.yml"
         ]  
       containers:
       - name: prometheus
@@ -104,9 +113,11 @@ spec:
           path: /mnt/disks/ssd0 #gke ssds
       - name: prometheus-executable
         emptyDir: {}
-      - name: config
-        hostPath:
-          path: /config-file 
+      - name: s3-config   # Define the Secret volume
+        secret:
+          secretName: s3-secret
+      - name: key
+        emptyDir: {}               
       terminationGracePeriodSeconds: 300
       nodeSelector:
         node-name: prometheus-{{ .PR_NUMBER }}
@@ -175,30 +186,39 @@ spec:
             cd /repo1 && \
             git fetch origin pull/{{ .PR_NUMBER }}/head:pr-branch && \
             git checkout pr-branch && \
-            cp  objectconfig.yml /mnt/repo/objectconfig.yml && \
+            cp  key.yml /config/key.yml && \
             rm -rf /repo1
         volumeMounts:
-        - name: config
-          mountPath: /mnt/repo
+        - name: key
+          mountPath: /config
       - name: data-downloader
         image: kushalshukla/writer
-        imagePullPolicy: Always        
+        imagePullPolicy: Always
+        env:
+        - name: ACCESS_KEY
+          value: "{{ .ACCESS_KEY }}"
+        - name: SECRET_KEY
+          value: "{{ .SECRET_KEY }}"  
+        - name: BUCKET_KEY
+          value: "{{ .BUCKET_KEY }}"                  
         volumeMounts:
         - name: instance-ssd
           mountPath: /data
-        - name: config
-          mountPath: /mnc                                   
+        - name: s3-config
+          mountPath: /config
+        - name: key
+          mountPath: /key                                               
         args: [
           "download",
         "--tsdb-path=/data", 
-        "--objstore.config-file=/mnc/objectconfig.yml",
-        "--key=gendata"
+        "--objstore.config-file=/config/object-config.yml",
+        "--key=/key/key.yml"
         ]            
       containers:
       - name: prometheus
         image: quay.io/prometheus/prometheus:{{ .RELEASE }}
         imagePullPolicy: Always
-        command: [ "/bin/prometheus"]
+        command: [ "/bin/prometheus" ]
         args: [
           "--web.external-url=http://{{ .DOMAIN_NAME }}/{{ .PR_NUMBER }}/prometheus-release",
           "--storage.tsdb.path=/prometheus",
@@ -224,7 +244,12 @@ spec:
           path: /mnt/disks/ssd0
       - name: config
         hostPath:
-          path: /object-config        
+          path: /object-config
+      - name: s3-config   # Define the Secret volume
+        secret:
+          secretName: s3-secret
+      - name: key
+        emptyDir: {}                              
       terminationGracePeriodSeconds: 300
       nodeSelector:
         node-name: prometheus-{{ .PR_NUMBER }}

tools/block-sync/README.md

@@ -41,10 +41,10 @@ The `download` command allows you to retrieve TSDB data from an object storage b
 The configuration file is essential for connecting to your object storage solution. Below are basic templates for different object storage systems.
 
 ```yaml
-type: s3
+type: s3, GCS , AZURE , etc.
 config:
   bucket: your-bucket-name
-  endpoint: https://your-minio-endpoint.com
+  endpoint: https://your-endpoint
   access_key: your-access-key
   secret_key: your-secret-key
   insecure: false  # Set to true if using HTTP instead of HTTPS

tools/block-sync/upload_download.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	"strings"
 
 	"github.com/go-kit/log"
 	"github.com/thanos-io/objstore"
@@ -42,10 +43,23 @@ func newStore(tsdbPath, objectConfig, objectKey string, logger *slog.Logger) (*S
 	if err != nil {
 		return nil, fmt.Errorf("failed to create bucket existence:%w", err)
 	}
+	key, err := os.ReadFile(objectKey)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read objectKey file: %w", err)
+	}
+
+	content := strings.TrimSpace(string(key))
+	var value string
+	if strings.HasPrefix(content, "key:") {
+		value = strings.TrimSpace(strings.TrimPrefix(content, "key:"))
+	} else {
+		return nil, fmt.Errorf("expected 'key:' prefix not found")
+	}
+
 	return &Store{
 		bucket:       bucket,
 		tsdbpath:     tsdbPath,
-		objectkey:    objectKey,
+		objectkey:    value,
 		objectconfig: objectConfig,
 		bucketlogger: logger,
 	}, nil

tools/load-generator/main.go

@@ -100,11 +100,10 @@ func NewQuerier(groupID int, target, prNumber string, qg QueryGroup) *Querier {
 
 	start := durationSeconds(qg.Start)
 	end := durationSeconds(qg.End)
-	fmt.Println(qtype, "HI from query ka type range or instant")
-	nodePort := 30198
-	url := fmt.Sprintf("http://%s:%d/%s/prometheus-%s/api/v1/query", domainName, nodePort, prNumber, target)
+
+	url := fmt.Sprintf("http://%s/%s/prometheus-%s/api/v1/query", domainName, prNumber, target)
 	if qtype == "range" {
-		url = fmt.Sprintf("http://%s:%d/%s/prometheus-%s/api/v1/query_range", domainName, nodePort, prNumber, target)
+		url = fmt.Sprintf("http://%s/%s/prometheus-%s/api/v1/query_range", domainName, prNumber, target)
 	}
 
 	return &Querier{
@@ -121,26 +120,28 @@ func NewQuerier(groupID int, target, prNumber string, qg QueryGroup) *Querier {
 	}
 }
 
-func (q *Querier) run(wg *sync.WaitGroup, baseTime *time.Time) {
+func (q *Querier) run(wg *sync.WaitGroup) {
 	defer wg.Done()
 	fmt.Printf("Running querier %s %s for %s\n", q.target, q.name, q.url)
 	time.Sleep(20 * time.Second)
+
 	for {
 		start := time.Now()
 
 		for _, query := range q.queries {
-			q.query(query.Expr, baseTime)
+			q.query(query.Expr)
 		}
-		*baseTime = baseTime.Add(2 * time.Minute)
+
 		wait := q.interval - time.Since(start)
 		if wait > 0 {
 			time.Sleep(wait)
 		}
 	}
 }
 
-func (q *Querier) query(expr string, baseTime *time.Time) {
+func (q *Querier) query(expr string) {
 	queryCount.WithLabelValues(q.target, q.name, expr, q.qtype).Inc()
+	start := time.Now()
 
 	req, err := http.NewRequest("GET", q.url, nil)
 	if err != nil {
@@ -152,12 +153,10 @@ func (q *Querier) query(expr string, baseTime *time.Time) {
 	qParams := req.URL.Query()
 	qParams.Set("query", expr)
 	if q.qtype == "range" {
-		qParams.Set("start", fmt.Sprintf("%d", int64(baseTime.Add(-q.start).Unix())))
-		qParams.Set("end", fmt.Sprintf("%d", int64(baseTime.Add(-q.end).Unix())))
+		qParams.Set("start", fmt.Sprintf("%d", int64(time.Now().Add(-q.start).Unix())))
+		qParams.Set("end", fmt.Sprintf("%d", int64(time.Now().Add(-q.end).Unix())))
 		qParams.Set("step", q.step)
 	}
-	fmt.Println(qParams["start"], "Hi from start Time")
-	fmt.Println(qParams["end"], "Hi from end End Time")
 	req.URL.RawQuery = qParams.Encode()
 
 	resp, err := http.DefaultClient.Do(req)
@@ -166,17 +165,11 @@ func (q *Querier) query(expr string, baseTime *time.Time) {
 		queryFailCount.WithLabelValues(q.target, q.name, expr, q.qtype).Inc()
 		return
 	}
-	// body, err := io.ReadAll(resp.Body)
-	// if err != nil {
-	// 	fmt.Println("Error reading response body:", err)
-	// 	return
-	// }
-	// fmt.Println(baseTime, "Hi from baseTime")
-	// fmt.Println("Response Body:", string(body))
 	defer resp.Body.Close()
 
-	duration := time.Since(*baseTime)
+	duration := time.Since(start)
 	queryDuration.WithLabelValues(q.target, q.name, expr, q.qtype).Observe(duration.Seconds())
+
 	if resp.StatusCode == http.StatusNotFound {
 		log.Printf("WARNING: GroupID#%d: Querier returned 404 for Prometheus instance %s.", q.groupID, q.url)
 		q.numberOfErrors++
@@ -227,12 +220,12 @@ func main() {
 	fmt.Println("Loaded configuration")
 
 	var wg sync.WaitGroup
-	baseTime := time.Date(2024, time.October, 23, 15, 34, 32, 541000000, time.UTC)
+
 	for i, group := range config.Querier.Groups {
 		wg.Add(1)
-		go NewQuerier(i, "pr", prNumber, group).run(&wg, &baseTime)
+		go NewQuerier(i, "pr", prNumber, group).run(&wg)
 		wg.Add(1)
-		go NewQuerier(i, "release", prNumber, group).run(&wg, &baseTime)
+		go NewQuerier(i, "release", prNumber, group).run(&wg)
 	}
 
 	prometheus.MustRegister(queryDuration, queryCount, queryFailCount)

tools/prometheus-builder/build.sh

@@ -5,8 +5,7 @@ DIR="/go/src/github.com/prometheus/prometheus"
 if [[ -z $PR_NUMBER || -z $VOLUME_DIR || -z $GITHUB_ORG || -z $GITHUB_REPO ]]; then
     echo "ERROR:: environment variables not set correctly"
     exit 1;
-fi
- 
+fi 
 # Clone the repository with a shallow clone
 echo ">> Cloning repository $GITHUB_ORG/$GITHUB_REPO (shallow clone)"
 if ! git clone --depth 1 https://github.com/$GITHUB_ORG/$GITHUB_REPO.git $DIR; then
@@ -25,7 +24,7 @@ fi
 git checkout pr-branch
 
 MKDIR="/config"
-cp  "$DIR/objectconfig.yml" "$MKDIR/objectconfig.yml"
+cp  "$DIR/key.yml" "$MKDIR/key.yml"
 
 echo ">> Creating prometheus binaries"
 if ! make build PROMU_BINARIES="prometheus"; then