add ability to run rule with remote write AND tsdb

Signed-off-by: Sam Kirsch <[email protected]>

cmd/thanos/rule.go

@@ -98,17 +98,18 @@ type ruleConfig struct {
 
 	rwConfig *extflag.PathOrContent
 
-	resendDelay        time.Duration
-	evalInterval       time.Duration
-	outageTolerance    time.Duration
-	forGracePeriod     time.Duration
-	ruleFiles          []string
-	objStoreConfig     *extflag.PathOrContent
-	dataDir            string
-	lset               labels.Labels
-	ignoredLabelNames  []string
-	storeRateLimits    store.SeriesSelectLimits
-	ruleConcurrentEval int64
+	resendDelay          time.Duration
+	evalInterval         time.Duration
+	outageTolerance      time.Duration
+	forGracePeriod       time.Duration
+	ruleFiles            []string
+	objStoreConfig       *extflag.PathOrContent
+	dataDir              string
+	lset                 labels.Labels
+	ignoredLabelNames    []string
+	storeRateLimits      store.SeriesSelectLimits
+	ruleConcurrentEval   int64
+	statelessModeEnabled bool
 
 	extendedFunctionsEnabled bool
 }
@@ -164,10 +165,12 @@ func registerRule(app *extkingpin.App) {
 
 	cmd.Flag("query.enable-x-functions", "Whether to enable extended rate functions (xrate, xincrease and xdelta). Only has effect when used with Thanos engine.").Default("false").BoolVar(&conf.extendedFunctionsEnabled)
 
-	conf.rwConfig = extflag.RegisterPathOrContent(cmd, "remote-write.config", "YAML config for the remote-write configurations, that specify servers where samples should be sent to (see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). This automatically enables stateless mode for ruler and no series will be stored in the ruler's TSDB. If an empty config (or file) is provided, the flag is ignored and ruler is run with its own TSDB.", extflag.WithEnvSubstitution())
+	conf.rwConfig = extflag.RegisterPathOrContent(cmd, "remote-write.config", "YAML config for the remote-write configurations, that specify servers where samples should be sent to (see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). If an empty config (or file) is provided, the flag is ignored and ruler is run with its own TSDB.", extflag.WithEnvSubstitution())
 
 	conf.objStoreConfig = extkingpin.RegisterCommonObjStoreFlags(cmd, "", false)
 
+	cmd.Flag("stateless", "Enable stateless mode for the ruler").Default("false").BoolVar(&conf.statelessModeEnabled)
+
 	reqLogConfig := extkingpin.RegisterRequestLoggingFlags(cmd)
 
 	var err error
@@ -475,6 +478,30 @@ func runRule(
 		return err
 	}
 
+	if !conf.statelessModeEnabled {
+		tsdbDB, err = tsdb.Open(conf.dataDir, log.With(logger, "component", "tsdb"), reg, tsdbOpts, nil)
+		if err != nil {
+			return errors.Wrap(err, "open TSDB")
+		}
+
+		level.Debug(logger).Log("msg", "removing storage lock file if any")
+		if err := removeLockfileIfAny(logger, conf.dataDir); err != nil {
+			return errors.Wrap(err, "remove storage lock files")
+		}
+
+		{
+			done := make(chan struct{})
+			g.Add(func() error {
+				<-done
+				return tsdbDB.Close()
+			}, func(error) {
+				close(done)
+			})
+		}
+		appendable = tsdbDB
+		queryable = tsdbDB
+	}
+
 	if len(rwCfgYAML) > 0 {
 		var rwCfg struct {
 			RemoteWriteConfigs []*config.RemoteWriteConfig `yaml:"remote_write,omitempty"`
@@ -496,38 +523,21 @@ func runRule(
 			return errors.Wrap(err, "applying config to remote storage")
 		}
 
-		agentDB, err = agent.Open(logger, reg, remoteStore, conf.dataDir, agentOpts)
-		if err != nil {
-			return errors.Wrap(err, "start remote write agent db")
+		var fanoutStore storage.Storage
+		if !conf.statelessModeEnabled {
+			fanoutStore = storage.NewFanout(logger, tsdbDB, remoteStore)
+		} else {
+			agentDB, err = agent.Open(logger, reg, remoteStore, conf.dataDir, agentOpts)
+			if err != nil {
+				return errors.Wrap(err, "start remote write agent db")
+			}
+			fanoutStore = storage.NewFanout(logger, agentDB, remoteStore)
 		}
-		fanoutStore := storage.NewFanout(logger, agentDB, remoteStore)
 		appendable = fanoutStore
 		// Use a separate queryable to restore the ALERTS firing states.
 		// We cannot use remoteStore directly because it uses remote read for
 		// query. However, remote read is not implemented in Thanos Receiver.
 		queryable = thanosrules.NewPromClientsQueryable(logger, queryClients, promClients, conf.query.httpMethod, conf.query.step, conf.ignoredLabelNames)
-	} else {
-		tsdbDB, err = tsdb.Open(conf.dataDir, log.With(logger, "component", "tsdb"), reg, tsdbOpts, nil)
-		if err != nil {
-			return errors.Wrap(err, "open TSDB")
-		}
-
-		level.Debug(logger).Log("msg", "removing storage lock file if any")
-		if err := removeLockfileIfAny(logger, conf.dataDir); err != nil {
-			return errors.Wrap(err, "remove storage lock files")
-		}
-
-		{
-			done := make(chan struct{})
-			g.Add(func() error {
-				<-done
-				return tsdbDB.Close()
-			}, func(error) {
-				close(done)
-			})
-		}
-		appendable = tsdbDB
-		queryable = tsdbDB
 	}
 
 	// Build the Alertmanager clients.