fix: Switch to an immediate drain strategy (#4230)

Switch to a immediate drain strategy * Ensures clients immediately receive a `connection: close` / `GOAWAY` instead of a probabilistic approach of receiving one b/w drain start and drain end (defaults to 600s). This should speed up shutdown with clients reconnecting to newer upgraded proxies. Fixes: #4205 Signed-off-by: Arko Dasgupta <[email protected]>
envoyproxy · Sep 13, 2024 · 14f687f · 14f687f
1 parent 6b8e37a
commit 14f687f
Show file tree

Hide file tree

Showing 163 changed files with 42 additions and 178 deletions.
diff --git a/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml b/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml
@@ -923,7 +923,6 @@ xds:
  statPrefix: http-10080
  useRemoteAddress: true
  name: default/eg/http
- drainType: MODIFY_ONLY
  name: default/eg/http
  perConnectionBufferLimitBytes: 32768
  - activeState:
@@ -991,7 +990,6 @@ xds:
  statPrefix: http-8080
  useRemoteAddress: true
  name: default/eg/grpc
- drainType: MODIFY_ONLY
  name: default/eg/grpc
  perConnectionBufferLimitBytes: 32768
  - activeState:
@@ -1014,7 +1012,6 @@ xds:
  socketAddress:
  address: 0.0.0.0
  portValue: 1234
- drainType: MODIFY_ONLY
  filterChains:
  - filters:
  - name: envoy.filters.network.tcp_proxy
@@ -1054,7 +1051,6 @@ xds:
  socketAddress:
  address: 0.0.0.0
  portValue: 8443
- drainType: MODIFY_ONLY
  filterChains:
  - filterChainMatch:
  serverNames:

diff --git a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.json b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.json
@@ -691,7 +691,6 @@
  ],
  "name": "default/eg/http"
  },
- "drainType": "MODIFY_ONLY",
  "name": "default/eg/http",
  "perConnectionBufferLimitBytes": 32768
  }
@@ -797,7 +796,6 @@
  ],
  "name": "default/eg/grpc"
  },
- "drainType": "MODIFY_ONLY",
  "name": "default/eg/grpc",
  "perConnectionBufferLimitBytes": 32768
  }
@@ -834,7 +832,6 @@
  "portValue": 1234
  }
  },
- "drainType": "MODIFY_ONLY",
  "filterChains": [
  {
  "filters": [
@@ -900,7 +897,6 @@
  "portValue": 8443
  }
  },
- "drainType": "MODIFY_ONLY",
  "filterChains": [
  {
  "filterChainMatch": {

diff --git a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.yaml b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.yaml
@@ -404,7 +404,6 @@ xds:
  statPrefix: http-10080
  useRemoteAddress: true
  name: default/eg/http
- drainType: MODIFY_ONLY
  name: default/eg/http
  perConnectionBufferLimitBytes: 32768
  - activeState:
@@ -472,7 +471,6 @@ xds:
  statPrefix: http-8080
  useRemoteAddress: true
  name: default/eg/grpc
- drainType: MODIFY_ONLY
  name: default/eg/grpc
  perConnectionBufferLimitBytes: 32768
  - activeState:
@@ -495,7 +493,6 @@ xds:
  socketAddress:
  address: 0.0.0.0
  portValue: 1234
- drainType: MODIFY_ONLY
  filterChains:
  - filters:
  - name: envoy.filters.network.tcp_proxy
@@ -535,7 +532,6 @@ xds:
  socketAddress:
  address: 0.0.0.0
  portValue: 8443
- drainType: MODIFY_ONLY
  filterChains:
  - filterChainMatch:
  serverNames:

diff --git a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.listener.yaml b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.listener.yaml
@@ -59,7 +59,6 @@ xds:
  statPrefix: http-10080
  useRemoteAddress: true
  name: default/eg/http
- drainType: MODIFY_ONLY
  name: default/eg/http
  perConnectionBufferLimitBytes: 32768
  - activeState:
@@ -127,7 +126,6 @@ xds:
  statPrefix: http-8080
  useRemoteAddress: true
  name: default/eg/grpc
- drainType: MODIFY_ONLY
  name: default/eg/grpc
  perConnectionBufferLimitBytes: 32768
  - activeState:
@@ -150,7 +148,6 @@ xds:
  socketAddress:
  address: 0.0.0.0
  portValue: 1234
- drainType: MODIFY_ONLY
  filterChains:
  - filters:
  - name: envoy.filters.network.tcp_proxy
@@ -190,7 +187,6 @@ xds:
  socketAddress:
  address: 0.0.0.0
  portValue: 8443
- drainType: MODIFY_ONLY
  filterChains:
  - filterChainMatch:
  serverNames:

diff --git a/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.json b/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.json
@@ -544,7 +544,6 @@
  ],
  "name": "envoy-gateway-system/eg/http"
  },
- "drainType": "MODIFY_ONLY",
  "name": "envoy-gateway-system/eg/http",
  "perConnectionBufferLimitBytes": 32768
  }

diff --git a/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.yaml b/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.yaml
@@ -320,7 +320,6 @@ xds:
  statPrefix: http-10080
  useRemoteAddress: true
  name: envoy-gateway-system/eg/http
- drainType: MODIFY_ONLY
  name: envoy-gateway-system/eg/http
  perConnectionBufferLimitBytes: 32768
  - '@type': type.googleapis.com/envoy.admin.v3.RoutesConfigDump

diff --git a/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.listener.yaml b/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.listener.yaml
@@ -76,6 +76,5 @@ xds:
  statPrefix: http-10080
  useRemoteAddress: true
  name: envoy-gateway-system/eg/http
- drainType: MODIFY_ONLY
  name: envoy-gateway-system/eg/http
  perConnectionBufferLimitBytes: 32768
diff --git a/internal/cmd/egctl/testdata/translate/out/no-service-cluster-ip.all.yaml b/internal/cmd/egctl/testdata/translate/out/no-service-cluster-ip.all.yaml
@@ -265,7 +265,6 @@ xds:
  statPrefix: http-10080
  useRemoteAddress: true
  name: envoy-gateway-system/eg/http
- drainType: MODIFY_ONLY
  name: envoy-gateway-system/eg/http
  perConnectionBufferLimitBytes: 32768
  - '@type': type.googleapis.com/envoy.admin.v3.RoutesConfigDump

diff --git a/internal/cmd/envoy/shutdown_manager.go b/internal/cmd/envoy/shutdown_manager.go
@@ -114,7 +114,7 @@ func shutdownReadyHandler(w http.ResponseWriter, readyTimeout time.Duration, rea
 }
 
 // Shutdown is called from a preStop hook on the shutdown-manager container where
-// it will initiate a graceful drain sequence on the Envoy proxy and block until
+// it will initiate a drain sequence on the Envoy proxy and block until
 // connections are drained or a timeout is exceeded.
 func Shutdown(drainTimeout time.Duration, minDrainDuration time.Duration, exitAtConnections int) error {
  startTime := time.Now()
@@ -125,19 +125,14 @@ func Shutdown(drainTimeout time.Duration, minDrainDuration time.Duration, exitAt
  logger = logging.FileLogger("/proc/1/fd/1", "shutdown-manager", egv1a1.LogLevelInfo)
  }
 
- logger.Info(fmt.Sprintf("initiating graceful drain with %.0f second minimum drain period and %.0f second timeout",
+ logger.Info(fmt.Sprintf("initiating drain with %.0f second minimum drain period and %.0f second timeout",
  minDrainDuration.Seconds(), drainTimeout.Seconds()))
 
  // Start failing active health checks
  if err := postEnvoyAdminAPI("healthcheck/fail"); err != nil {
  logger.Error(err, "error failing active health checks")
  }
 
- // Initiate graceful drain sequence
- if err := postEnvoyAdminAPI("drain_listeners?graceful&skip_exit"); err != nil {
- logger.Error(err, "error initiating graceful drain")
- }
-
  // Poll total connections from Envoy admin API until minimum drain period has
  // been reached and total connections reaches threshold or timeout is exceeded
  for {
@@ -154,10 +149,10 @@ func Shutdown(drainTimeout time.Duration, minDrainDuration time.Duration, exitAt
  }
 
  if elapsedTime > drainTimeout {
- logger.Info("graceful drain sequence timeout exceeded")
+ logger.Info("drain sequence timeout exceeded")
  break
  } else if allowedToExit && conn != nil && *conn <= exitAtConnections {
- logger.Info("graceful drain sequence completed")
+ logger.Info("drain sequence completed")
  break
  }
 

diff --git a/internal/infrastructure/kubernetes/proxy/resource.go b/internal/infrastructure/kubernetes/proxy/resource.go
@@ -171,6 +171,7 @@ func expectedProxyContainers(infra *ir.ProxyInfra,
  fmt.Sprintf("--config-yaml %s", bootstrapConfigurations),
  fmt.Sprintf("--log-level %s", logging.DefaultEnvoyProxyLoggingLevel()),
  "--cpuset-threads",
+ "--drain-strategy immediate",
  }
 
  if infra.Config != nil &&

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/component-level.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/component-level.yaml
@@ -40,6 +40,7 @@ spec:
  - --config-yaml test bootstrap config
  - --log-level error
  - --cpuset-threads
+ - --drain-strategy immediate
  - --component-log-level filter:info
  command:
  - envoy

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/custom.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/custom.yaml
@@ -224,6 +224,7 @@ spec:
  value: 0.98
  - --log-level warn
  - --cpuset-threads
+ - --drain-strategy immediate
  command:
  - envoy
  env:

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/default-env.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/default-env.yaml
@@ -223,6 +223,7 @@ spec:
  value: 0.98
  - --log-level warn
  - --cpuset-threads
+ - --drain-strategy immediate
  command:
  - envoy
  env:

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/default.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/default.yaml
@@ -208,6 +208,7 @@ spec:
  max_active_downstream_connections: 50000
  - --log-level warn
  - --cpuset-threads
+ - --drain-strategy immediate
  command:
  - envoy
  env:

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/disable-prometheus.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/disable-prometheus.yaml
@@ -182,6 +182,7 @@ spec:
  max_active_downstream_connections: 50000
  - --log-level warn
  - --cpuset-threads
+ - --drain-strategy immediate
  command:
  - envoy
  env:

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/extension-env.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/extension-env.yaml
@@ -223,6 +223,7 @@ spec:
  value: 0.98
  - --log-level warn
  - --cpuset-threads
+ - --drain-strategy immediate
  command:
  - envoy
  env:

diff --git a/.../infrastructure/kubernetes/proxy/testdata/daemonsets/override-labels-and-annotations.yaml b/.../infrastructure/kubernetes/proxy/testdata/daemonsets/override-labels-and-annotations.yaml
@@ -217,6 +217,7 @@ spec:
  max_active_downstream_connections: 50000
  - --log-level warn
  - --cpuset-threads
+ - --drain-strategy immediate
  command:
  - envoy
  env:

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/patch-daemonset.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/patch-daemonset.yaml
@@ -208,6 +208,7 @@ spec:
  max_active_downstream_connections: 50000
  - --log-level warn
  - --cpuset-threads
+ - --drain-strategy immediate
  command:
  - envoy
  env:

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/shutdown-manager.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/shutdown-manager.yaml
@@ -208,6 +208,7 @@ spec:
  max_active_downstream_connections: 50000
  - --log-level warn
  - --cpuset-threads
+ - --drain-strategy immediate
  - --drain-time-s 30
  command:
  - envoy

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/volumes.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/volumes.yaml
@@ -223,6 +223,7 @@ spec:
  value: 0.98
  - --log-level warn
  - --cpuset-threads
+ - --drain-strategy immediate
  command:
  - envoy
  env:

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-annotations.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-annotations.yaml
@@ -213,6 +213,7 @@ spec:
  max_active_downstream_connections: 50000
  - --log-level warn
  - --cpuset-threads
+ - --drain-strategy immediate
  command:
  - envoy
  env:

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-concurrency.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-concurrency.yaml
@@ -40,6 +40,7 @@ spec:
  - --config-yaml test bootstrap config
  - --log-level warn
  - --cpuset-threads
+ - --drain-strategy immediate
  - --concurrency 4
  command:
  - envoy

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-extra-args.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-extra-args.yaml
@@ -208,6 +208,7 @@ spec:
  max_active_downstream_connections: 50000
  - --log-level warn
  - --cpuset-threads
+ - --drain-strategy immediate
  - --key1 val1
  - --key2 val2
  command:

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-image-pull-secrets.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-image-pull-secrets.yaml
@@ -208,6 +208,7 @@ spec:
  max_active_downstream_connections: 50000
  - --log-level warn
  - --cpuset-threads
+ - --drain-strategy immediate
  command:
  - envoy
  env:

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-name.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-name.yaml
@@ -208,6 +208,7 @@ spec:
  max_active_downstream_connections: 50000
  - --log-level warn
  - --cpuset-threads
+ - --drain-strategy immediate
  command:
  - envoy
  env:

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-node-selector.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-node-selector.yaml
@@ -208,6 +208,7 @@ spec:
  max_active_downstream_connections: 50000
  - --log-level warn
  - --cpuset-threads
+ - --drain-strategy immediate
  command:
  - envoy
  env:

diff --git a/...infrastructure/kubernetes/proxy/testdata/daemonsets/with-topology-spread-constraints.yaml b/...infrastructure/kubernetes/proxy/testdata/daemonsets/with-topology-spread-constraints.yaml
@@ -208,6 +208,7 @@ spec:
  max_active_downstream_connections: 50000
  - --log-level warn
  - --cpuset-threads
+ - --drain-strategy immediate
  command:
  - envoy
  env:

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml
@@ -44,6 +44,7 @@ spec:
  - --config-yaml test bootstrap config
  - --log-level warn
  - --cpuset-threads
+ - --drain-strategy immediate
  command:
  - envoy
  env:

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/component-level.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/component-level.yaml
@@ -44,6 +44,7 @@ spec:
  - --config-yaml test bootstrap config
  - --log-level error
  - --cpuset-threads
+ - --drain-strategy immediate
  - --component-log-level filter:info
  command:
  - envoy

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml
@@ -229,6 +229,7 @@ spec:
  value: 0.98
  - --log-level warn
  - --cpuset-threads
+ - --drain-strategy immediate
  command:
  - envoy
  env:

diff --git a/...rnal/infrastructure/kubernetes/proxy/testdata/deployments/custom_with_initcontainers.yaml b/...rnal/infrastructure/kubernetes/proxy/testdata/deployments/custom_with_initcontainers.yaml
@@ -229,6 +229,7 @@ spec:
  value: 0.98
  - --log-level warn
  - --cpuset-threads
+ - --drain-strategy immediate
  command:
  - envoy
  env:

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml
@@ -228,6 +228,7 @@ spec:
  value: 0.98
  - --log-level warn
  - --cpuset-threads
+ - --drain-strategy immediate
  command:
  - envoy
  env:

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml
@@ -212,6 +212,7 @@ spec:
  max_active_downstream_connections: 50000
  - --log-level warn
  - --cpuset-threads
+ - --drain-strategy immediate
  command:
  - envoy
  env: