Feat/rpc helper

config/settings.go

@@ -6,6 +6,10 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"time"
+
+	rpchelper "github.com/powerloom/go-rpc-helper"
+	"github.com/powerloom/go-rpc-helper/reporting"
 
 	"github.com/ethereum/go-ethereum/common"
 )
@@ -18,6 +22,15 @@ type DataMarketMigrationEntry struct {
 }
 
 type Settings struct {
+	// RPC Helper Configuration
+	RPCNodes        []string `json:"rpc_nodes"`
+	ArchiveRPCNodes []string `json:"archive_rpc_nodes"`
+	MaxRetries      int      `json:"max_retries"`
+	RetryDelayMs    int      `json:"retry_delay_ms"`
+	MaxRetryDelayS  int      `json:"max_retry_delay_s"`
+	RequestTimeoutS int      `json:"request_timeout_s"`
+
+	// Legacy fields (keeping for backward compatibility during transition)
 	ClientUrl                   string
 	ContractAddress             string
 	RedisHost                   string
@@ -53,17 +66,57 @@ type Settings struct {
 }
 
 func LoadConfig() {
+	// Parse RPC nodes from environment variable
+	rpcNodesStr := getEnv("RPC_NODES", "[]")
+	var rpcNodes []string
+	err := json.Unmarshal([]byte(rpcNodesStr), &rpcNodes)
+	if err != nil {
+		log.Fatalf("Failed to parse RPC_NODES environment variable: %v", err)
+	}
+	if len(rpcNodes) == 0 {
+		// Fallback to legacy PROST_RPC_URL for backward compatibility
+		legacyRPCURL := getEnv("PROST_RPC_URL", "")
+		if legacyRPCURL != "" {
+			rpcNodes = []string{legacyRPCURL}
+		} else {
+			log.Fatalf("RPC_NODES environment variable has an empty array and no PROST_RPC_URL fallback")
+		}
+	}
+
+	// Clean quotes from RPC node URLs
+	for i, url := range rpcNodes {
+		rpcNodes[i] = strings.Trim(url, "\"")
+	}
+
+	// Parse archive RPC nodes from environment variable (optional)
+	archiveRPCNodesStr := getEnv("ARCHIVE_RPC_NODES", "[]")
+	var archiveRPCNodes []string
+	err = json.Unmarshal([]byte(archiveRPCNodesStr), &archiveRPCNodes)
+	if err != nil {
+		log.Fatalf("Failed to parse ARCHIVE_RPC_NODES environment variable: %v", err)
+	}
+
+	// Clean quotes from archive RPC node URLs
+	for i, url := range archiveRPCNodes {
+		archiveRPCNodes[i] = strings.Trim(url, "\"")
+	}
+
 	dataMarketAddresses := getEnv("DATA_MARKET_ADDRESSES", "[]")
 	dataMarketAddressesList := []string{}
 
-	err := json.Unmarshal([]byte(dataMarketAddresses), &dataMarketAddressesList)
+	err = json.Unmarshal([]byte(dataMarketAddresses), &dataMarketAddressesList)
 	if err != nil {
 		log.Fatalf("Failed to parse DATA_MARKET_ADDRESSES environment variable: %v", err)
 	}
 	if len(dataMarketAddressesList) == 0 {
 		log.Fatalf("DATA_MARKET_ADDRESSES environment variable has an empty array")
 	}
 
+	// Clean quotes from data market addresses
+	for i, addr := range dataMarketAddressesList {
+		dataMarketAddressesList[i] = strings.Trim(addr, "\"")
+	}
+
 	periodicEligibleCountAlerts, periodicEligibleCountAlertsErr := strconv.ParseBool(getEnv("PERIODIC_ELIGIBLE_COUNT_ALERTS", "true"))
 	if periodicEligibleCountAlertsErr != nil {
 		log.Fatalf("Failed to parse PERIODIC_ELIGIBLE_COUNT_ALERTS environment variable: %v", periodicEligibleCountAlertsErr)
@@ -93,8 +146,17 @@ func LoadConfig() {
 	}
 
 	config := Settings{
+		// RPC Helper Configuration
+		RPCNodes:        rpcNodes,
+		ArchiveRPCNodes: archiveRPCNodes,
+		MaxRetries:      getEnvInt("RPC_MAX_RETRIES", 3),
+		RetryDelayMs:    getEnvInt("RPC_RETRY_DELAY_MS", 500),
+		MaxRetryDelayS:  getEnvInt("RPC_MAX_RETRY_DELAY_S", 30),
+		RequestTimeoutS: getEnvInt("RPC_REQUEST_TIMEOUT_S", 30),
+
+		// Legacy configuration (keeping for backward compatibility)
 		ClientUrl:                   getEnv("PROST_RPC_URL", ""),
-		ContractAddress:             getEnv("PROTOCOL_STATE_CONTRACT", ""),
+		ContractAddress:             strings.Trim(getEnv("PROTOCOL_STATE_CONTRACT", ""), "\""),
 		RedisHost:                   getEnv("REDIS_HOST", ""),
 		RedisPort:                   getEnv("REDIS_PORT", ""),
 		RedisDB:                     getEnv("REDIS_DB", ""),
@@ -224,3 +286,52 @@ func getEnv(key, defaultValue string) string {
 	}
 	return value
 }
+
+func getEnvInt(key string, defaultValue int) int {
+	value := getEnv(key, "")
+	if value == "" {
+		return defaultValue
+	}
+	intValue, err := strconv.Atoi(value)
+	if err != nil {
+		log.Fatalf("Failed to parse %s environment variable: %v", key, err)
+	}
+	return intValue
+}
+
+func (s *Settings) ToRPCConfig() *rpchelper.RPCConfig {
+	config := &rpchelper.RPCConfig{
+		Nodes: func() []rpchelper.NodeConfig {
+			var nodes []rpchelper.NodeConfig
+			for _, url := range s.RPCNodes {
+				nodes = append(nodes, rpchelper.NodeConfig{URL: url})
+			}
+			return nodes
+		}(),
+		ArchiveNodes: func() []rpchelper.NodeConfig {
+			var nodes []rpchelper.NodeConfig
+			for _, url := range s.ArchiveRPCNodes {
+				nodes = append(nodes, rpchelper.NodeConfig{URL: url})
+			}
+			return nodes
+		}(),
+		MaxRetries:     s.MaxRetries,
+		RetryDelay:     time.Duration(s.RetryDelayMs) * time.Millisecond,
+		MaxRetryDelay:  time.Duration(s.MaxRetryDelayS) * time.Second,
+		RequestTimeout: time.Duration(s.RequestTimeoutS) * time.Second,
+	}
+
+	// Configure webhook if SlackReportingUrl is provided
+	if s.SlackReportingUrl != "" {
+		log.Printf("Configuring webhook alerts with URL: %s", s.SlackReportingUrl)
+		config.WebhookConfig = &reporting.WebhookConfig{
+			URL:     s.SlackReportingUrl,
+			Timeout: 30 * time.Second,
+			Retries: 3,
+		}
+	} else {
+		log.Printf("No webhook URL configured - SLACK_REPORTING_URL environment variable not set")
+	}
+
+	return config
+}

go.mod

@@ -1,19 +1,23 @@
 module submission-sequencer-collector
 
-go 1.23
+go 1.23.0
+
+toolchain go1.23.4
 
 require (
 	github.com/ethereum/go-ethereum v1.14.7
 	github.com/gorilla/handlers v1.5.2
+	github.com/powerloom/go-rpc-helper v1.0.5-0.20250726065922-28883c22bd97
 	github.com/sirupsen/logrus v1.9.3
-	github.com/stretchr/testify v1.9.0
+	github.com/stretchr/testify v1.10.0
 	github.com/swaggo/swag v1.16.4
 	google.golang.org/protobuf v1.34.2
 )
 
 require (
 	github.com/KyleBanks/depth v1.2.1 // indirect
 	github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
@@ -61,9 +65,9 @@ require (
 	github.com/swaggo/http-swagger v1.3.4
 	github.com/tklauser/go-sysconf v0.3.14 // indirect
 	github.com/tklauser/numcpus v0.9.0 // indirect
-	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/crypto v0.35.0 // indirect
 	golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect
-	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/sys v0.30.0 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
 )
 

go.sum

@@ -18,6 +18,8 @@ github.com/bits-and-blooms/bitset v1.14.3 h1:Gd2c8lSNf9pKXom5JtD7AaKO8o7fGQ2LtFj
 github.com/bits-and-blooms/bitset v1.14.3/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
+github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
 github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -156,6 +158,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/powerloom/go-rpc-helper v1.0.5-0.20250726065922-28883c22bd97 h1:znQ7p+uicav29JDXALd/bOnDSv+Wg5b4uXRZpwjizbs=
+github.com/powerloom/go-rpc-helper v1.0.5-0.20250726065922-28883c22bd97/go.mod h1:RxQj0kBBezXUWdH3j7rCXcDLXNwVlElT31qCKPNUFUI=
 github.com/prometheus/client_golang v1.12.0 h1:C+UIj/QWtmqY13Arb8kwMt5j34/0Z2iKamrJ+ryC0Gg=
 github.com/prometheus/client_golang v1.12.0/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a h1:CmF68hwI0XsOQ5UwlBopMi2Ow4Pbg32akc4KIVCOm+Y=
@@ -180,8 +184,8 @@ github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobt
 github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/supranational/blst v0.3.13 h1:AYeSxdOMacwu7FBmpfloBz5pbFXDmJL33RuwnKtmTjk=
 github.com/supranational/blst v0.3.13/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw=
 github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE=
@@ -209,8 +213,8 @@ github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
-golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
+golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs=
+golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
 golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY=
 golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -235,17 +239,17 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
-golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
-golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
+golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

pkgs/constants.go

@@ -14,7 +14,6 @@ const (
 	HandleDayTransition             = "SequencerEventCollector: HandleDayTransition"
 	TriggerBatchPreparation         = "SequencerEventCollector: TriggerBatchPreparation"
 	CheckAndTriggerBatchPreparation = "SequencerEventCollector: CheckAndTriggerBatchPreparation"
-	MustQuery                       = "SequencerEventCollector: MustQuery"
 )
 
 // General Key Constants

pkgs/prost/cleanup.go

@@ -113,6 +113,7 @@ func CleanupSubmissionDumpForAllSlots(parentCtx context.Context, dataMarketAddr
 		log.Errorf("Failed to parse node count %s: %v", nodeCountStr, err)
 		return err
 	}
+	// TODO: remove unnecessary cleanup since this key is not used anymore
 	for slotId := 1; slotId <= nodeCount; slotId++ {
 		log.Debugf("Cleaning up old submission dump for slot %d for data market %s", slotId, dataMarketAddr)
 		keyPattern := fmt.Sprintf("snapshotter:%s:*:%d.slot_submissions", strings.ToLower(dataMarketAddr), slotId)