Merge pull request #73 from deckhouse/add-checks-for-controller

[controller] add liveness and readiness checks
deckhouse · Jul 13, 2024 · a8818d2 · a8818d2
2 parents 1702489 + 361cc1b
commit a8818d2
Show file tree

Hide file tree

Showing 6 changed files with 69 additions and 21 deletions.
diff --git a/images/agent/cmd/main.go b/images/agent/cmd/main.go
@@ -93,9 +93,10 @@ func main() {
 	log.Info("[main] successfully read scheme CR")
 
 	managerOpts := manager.Options{
-		Scheme:  scheme,
-		Logger:  log.GetLogger(),
-		Metrics: server.Options{BindAddress: cfgParams.MetricsPort},
+		Scheme:                 scheme,
+		Logger:                 log.GetLogger(),
+		Metrics:                server.Options{BindAddress: cfgParams.MetricsPort},
+		HealthProbeBindAddress: cfgParams.HealthProbeBindAddress,
 	}
 
 	mgr, err := manager.New(kConfig, managerOpts)

diff --git a/images/agent/config/config.go b/images/agent/config/config.go
@@ -30,13 +30,15 @@ import (
 )
 
 const (
-	ScanInterval        = "SCAN_INTERVAL"
-	NodeName            = "NODE_NAME"
-	LogLevel            = "LOG_LEVEL"
-	MetricsPort         = "METRICS_PORT"
-	MachineID           = "MACHINE_ID"
-	ThrottleInterval    = "THROTTLER_INTERVAL"
-	CmdDeadlineDuration = "CMD_DEADLINE_DURATION"
+	ScanInterval                         = "SCAN_INTERVAL"
+	NodeName                             = "NODE_NAME"
+	LogLevel                             = "LOG_LEVEL"
+	MetricsPort                          = "METRICS_PORT"
+	MachineID                            = "MACHINE_ID"
+	ThrottleInterval                     = "THROTTLER_INTERVAL"
+	CmdDeadlineDuration                  = "CMD_DEADLINE_DURATION"
+	DefaultHealthProbeBindAddressEnvName = "HEALTH_PROBE_BIND_ADDRESS"
+	DefaultHealthProbeBindAddress        = ":8081"
 )
 
 type Options struct {
@@ -49,6 +51,7 @@ type Options struct {
 	LLVRequeueIntervalSec      time.Duration
 	ThrottleIntervalSec        time.Duration
 	CmdDeadlineDurationSec     time.Duration
+	HealthProbeBindAddress     string
 }
 
 func NewConfig() (*Options, error) {
@@ -77,6 +80,11 @@ func NewConfig() (*Options, error) {
 		opts.MetricsPort = ":9695"
 	}
 
+	opts.HealthProbeBindAddress = os.Getenv(DefaultHealthProbeBindAddressEnvName)
+	if opts.HealthProbeBindAddress == "" {
+		opts.HealthProbeBindAddress = DefaultHealthProbeBindAddress
+	}
+
 	scanInt := os.Getenv(ScanInterval)
 	if scanInt == "" {
 		opts.BlockDeviceScanIntervalSec = 5 * time.Second

diff --git a/images/sds-health-watcher-controller/cmd/main.go b/images/sds-health-watcher-controller/cmd/main.go
@@ -88,9 +88,10 @@ func main() {
 	log.Info("[main] successfully read scheme CR")
 
 	managerOpts := manager.Options{
-		Scheme:  scheme,
-		Logger:  log.GetLogger(),
-		Metrics: server.Options{BindAddress: cfgParams.MetricsPort},
+		Scheme:                 scheme,
+		Logger:                 log.GetLogger(),
+		Metrics:                server.Options{BindAddress: cfgParams.MetricsPort},
+		HealthProbeBindAddress: cfgParams.HealthProbeBindAddress,
 	}
 
 	mgr, err := manager.New(kConfig, managerOpts)

diff --git a/images/sds-health-watcher-controller/config/config.go b/images/sds-health-watcher-controller/config/config.go
@@ -26,17 +26,20 @@ import (
 )
 
 const (
-	ScanInterval = "SCAN_INTERVAL"
-	NodeName     = "NODE_NAME"
-	LogLevel     = "LOG_LEVEL"
-	MetricsPort  = "METRICS_PORT"
+	ScanInterval                         = "SCAN_INTERVAL"
+	NodeName                             = "NODE_NAME"
+	LogLevel                             = "LOG_LEVEL"
+	MetricsPort                          = "METRICS_PORT"
+	DefaultHealthProbeBindAddressEnvName = "HEALTH_PROBE_BIND_ADDRESS"
+	DefaultHealthProbeBindAddress        = ":8081"
 )
 
 type Options struct {
-	Loglevel        logger.Verbosity
-	MetricsPort     string
-	ScanIntervalSec time.Duration
-	NodeName        string
+	Loglevel               logger.Verbosity
+	MetricsPort            string
+	ScanIntervalSec        time.Duration
+	NodeName               string
+	HealthProbeBindAddress string
 }
 
 func NewConfig() (*Options, error) {
@@ -54,6 +57,11 @@ func NewConfig() (*Options, error) {
 		opts.MetricsPort = ":9695"
 	}
 
+	opts.HealthProbeBindAddress = os.Getenv(DefaultHealthProbeBindAddressEnvName)
+	if opts.HealthProbeBindAddress == "" {
+		opts.HealthProbeBindAddress = DefaultHealthProbeBindAddress
+	}
+
 	scanInt := os.Getenv(ScanInterval)
 	if scanInt == "" {
 		opts.ScanIntervalSec = 5 * time.Second

diff --git a/templates/agent/daemonset.yaml b/templates/agent/daemonset.yaml
@@ -69,6 +69,21 @@ spec:
       - name: sds-node-configurator-agent
         image: {{ include "helm_lib_module_image" (list . "agent") }}
         imagePullPolicy: IfNotPresent
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: 8081
+            scheme: HTTP
+          initialDelaySeconds: 5
+          failureThreshold: 2
+          periodSeconds: 1
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8081
+            scheme: HTTP
+          periodSeconds: 1
+          failureThreshold: 3
         securityContext:
           privileged: true
           seLinuxOptions:

diff --git a/templates/sds-health-watcher-controller/deployment.yaml b/templates/sds-health-watcher-controller/deployment.yaml
@@ -68,6 +68,21 @@ spec:
         - name: sds-health-watcher-controller
           image: {{ include "helm_lib_module_image" (list . "sdsHealthWatcherController") }}
           imagePullPolicy: IfNotPresent
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: 8081
+              scheme: HTTP
+            initialDelaySeconds: 5
+            failureThreshold: 2
+            periodSeconds: 1
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 8081
+              scheme: HTTP
+            periodSeconds: 1
+            failureThreshold: 3
           resources:
             requests:
               {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 14 }}