internal/https: add health checking support and use it

CL 454935 broke the Kubernetes ingress by requiring IAP on health
checks. Move /healthz handling into internal/https, where it
automatically bypasses authentication and removes some duplicate trivial
implementations.

Unfortunately, GKE is not capable of inferring health check parameters
from a multi-container pod like relui, so we have to change our
BackendConfig. That sets off a yak shave -- I made the questionable
decision to use the same backend for all our IAP services, and the
coordinator doesn't currently support /healthz. Split all them up and
delete the devapp configuration I was using for testing way back in the
day.

Change-Id: I45e866d30508a07e9a805de70af731dd64c22d7f
Reviewed-on: https://go-review.googlesource.com/c/build/+/455215
Reviewed-by: Dmitri Shuralyov <[email protected]>
Auto-Submit: Heschi Kreinick <[email protected]>
Run-TryBot: Heschi Kreinick <[email protected]>
Reviewed-by: Dmitri Shuralyov <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>

Author: heschi

cmd/coordinator/deployment-prod.yaml

@@ -67,7 +67,7 @@ metadata:
   namespace: prod
   name: coordinator-internal-iap
   annotations:
-    cloud.google.com/backend-config: '{"default": "build-ingress-iap-backend"}'
+    cloud.google.com/backend-config: '{"default": "coordinator-iap-backend"}'
     cloud.google.com/neg: '{"ingress": false}'
     cloud.google.com/app-protocols: '{"https":"HTTP2"}'
 spec:
@@ -95,3 +95,15 @@ spec:
   selector:
     app: coordinator
   type: NodePort
+---
+apiVersion: cloud.google.com/v1
+kind: BackendConfig
+metadata:
+  namespace: prod
+  name: coordinator-iap-backend
+spec:
+  iap:
+    enabled: true
+    oauthclientCredentials:
+      secretName: iap-oauth
+  timeoutSec: 86400  # For long-running gomote RPCs. See https://go.dev/issue/56423.

cmd/relui/deployment-prod.yaml

@@ -43,6 +43,11 @@ spec:
             - "--serving-files-base=gs://golang"
             - "--edge-cache-url=https://dl.google.com/go"
             - "--website-upload-url=https://go.dev/dl/upload"
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: 444
+              scheme: HTTPS
           ports:
             - containerPort: 444
           env:
@@ -100,7 +105,7 @@ metadata:
   namespace: prod
   name: relui-internal
   annotations:
-    cloud.google.com/backend-config: '{"default": "build-ingress-iap-backend"}'
+    cloud.google.com/backend-config: '{"default": "relui-iap-backend"}'
     cloud.google.com/neg: '{"ingress": false}'
     cloud.google.com/app-protocols: '{"https":"HTTP2"}'
 spec:
@@ -111,3 +116,19 @@ spec:
   selector:
     app: relui
   type: NodePort
+---
+apiVersion: cloud.google.com/v1
+kind: BackendConfig
+metadata:
+  namespace: prod
+  name: relui-iap-backend
+spec:
+  iap:
+    enabled: true
+    oauthclientCredentials:
+      secretName: iap-oauth
+  healthCheck:
+    timeoutSec: 10
+    checkIntervalSec: 15
+    type: HTTPS
+    requestPath: /healthz

deploy/build-ingress.yaml

@@ -17,13 +17,6 @@ spec:
   - host: dev-test.golang.org
     http:
       paths:
-      - pathType: ImplementationSpecific
-        path: /owners
-        backend:
-          service:
-            name: devapp-internal-iap
-            port:
-              number: 444
       - pathType: ImplementationSpecific
         path: /*
         backend:
@@ -137,26 +130,6 @@ spec:
     enabled: true
     responseCodeName: FOUND
 ---
-apiVersion: cloud.google.com/v1
-kind: BackendConfig
-metadata:
-  namespace: prod
-  name: build-ingress-iap-backend
-spec:
-  iap:
-    enabled: true
-    oauthclientCredentials:
-      secretName: iap-oauth
-  timeoutSec: 86400  # For long-running gomote RPCs. See https://go.dev/issue/56423.
----
-apiVersion: cloud.google.com/v1
-kind: BackendConfig
-metadata:
-  namespace: prod
-  name: build-ingress-maintnerd-backend
-spec:
-  timeoutSec: 60  # For long-poll support on the /logs endpoint. See go.dev/issue/53569.
----
 apiVersion: networking.gke.io/v1
 kind: ManagedCertificate
 metadata:

devapp/deployment-prod.yaml

@@ -44,24 +44,6 @@ spec:
 ---
 apiVersion: v1
 kind: Service
-metadata:
-  namespace: prod
-  name: devapp-internal-iap
-  annotations:
-    cloud.google.com/backend-config: '{"default": "build-ingress-iap-backend"}'
-    cloud.google.com/neg: '{"ingress": false}'
-    cloud.google.com/app-protocols: '{"https":"HTTP2"}'
-spec:
-  ports:
-    - port: 444
-      targetPort: 444
-      name: https
-  selector:
-    app: devapp
-  type: NodePort
----
-apiVersion: v1
-kind: Service
 metadata:
   namespace: prod
   name: devapp-internal

devapp/server.go

@@ -64,7 +64,6 @@ func newServer(mux *http.ServeMux, staticDir, templateDir string, reloadTmpls bo
 		userMapping: map[int]*maintner.GitHubUser{},
 	}
 	s.mux.Handle("/", http.FileServer(http.Dir(s.staticDir)))
-	s.mux.HandleFunc("/healthz", handleHealthz)
 	s.mux.HandleFunc("/favicon.ico", s.handleFavicon)
 	s.mux.HandleFunc("/release", s.withTemplate("/release.tmpl", s.handleRelease))
 	s.mux.HandleFunc("/reviews", s.withTemplate("/reviews.tmpl", s.handleReviews))
@@ -203,11 +202,6 @@ func (s *server) filteredHelpWantedIssues(pkgs ...string) []issueData {
 	return issues
 }
 
-func handleHealthz(w http.ResponseWriter, r *http.Request) {
-	w.WriteHeader(http.StatusOK)
-	w.Write([]byte("ok"))
-}
-
 func (s *server) handleFavicon(w http.ResponseWriter, r *http.Request) {
 	// Need to specify content type for consistent tests, without this it's
 	// determined from mime.types on the box the test is running on

internal/https/https.go

@@ -34,6 +34,8 @@ type Options struct {
 	SelfSignedAddr string
 	// If non-empty, listen on this address and serve HTTP.
 	HTTPAddr string
+	// If non-empty, respond unconditionally with 200 OK to requests on this path.
+	HealthPath string
 }
 
 var DefaultOptions = &Options{}
@@ -47,6 +49,7 @@ func RegisterFlags(set *flag.FlagSet) {
 	set.StringVar(&DefaultOptions.AutocertAddr, "listen-https-autocert", "", "if non-empty, listen on this address and serve HTTPS using a Let's Encrypt cert stored in autocert-bucket")
 	set.StringVar(&DefaultOptions.SelfSignedAddr, "listen-https-selfsigned", "", "if non-empty, listen on this address and serve HTTPS using a self-signed cert")
 	set.StringVar(&DefaultOptions.HTTPAddr, "listen-http", "", "if non-empty, listen on this address and serve HTTP")
+	set.StringVar(&DefaultOptions.HealthPath, "health-path", "/healthz", "if non-empty, respond unconditionally with 200 OK to requests on this path")
 }
 
 // ListenAndServe runs the servers configured by DefaultOptions. It always
@@ -60,6 +63,18 @@ func ListenAndServe(ctx context.Context, handler http.Handler) error {
 func ListenAndServeOpts(ctx context.Context, handler http.Handler, opts *Options) error {
 	errc := make(chan error, 3)
 
+	if opts.HealthPath != "" {
+		wrapped := handler
+		handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if r.URL.Path == opts.HealthPath {
+				w.WriteHeader(http.StatusOK)
+				w.Write([]byte("ok"))
+			} else {
+				wrapped.ServeHTTP(w, r)
+			}
+		})
+	}
+
 	if opts.HTTPAddr != "" {
 		server := &http.Server{Addr: opts.HTTPAddr, Handler: handler}
 		defer server.Close()

maintner/maintnerd/deployment-prod.yaml

@@ -74,3 +74,11 @@ spec:
   selector:
     app: maintnerd
   type: NodePort
+---
+apiVersion: cloud.google.com/v1
+kind: BackendConfig
+metadata:
+  namespace: prod
+  name: build-ingress-maintnerd-backend
+spec:
+  timeoutSec: 60  # For long-poll support on the /logs endpoint. See go.dev/issue/53569.

perf/app/app.go

@@ -59,7 +59,6 @@ func (a *App) RegisterOnMux(mux *http.ServeMux) {
 	mux.HandleFunc("/search", a.search)
 	mux.HandleFunc("/compare", a.compare)
 	mux.HandleFunc("/cron/syncinflux", a.syncInflux)
-	mux.HandleFunc("/healthz", a.healthz)
 	a.dashboardRegisterOnMux(mux)
 }
 
@@ -82,9 +81,3 @@ func (a *App) search(w http.ResponseWriter, r *http.Request) {
 	//q := r.Form.Get("q")
 	a.compare(w, r)
 }
-
-// healthz handles /healthz.
-func (a *App) healthz(w http.ResponseWriter, r *http.Request) {
-	w.WriteHeader(http.StatusOK)
-	w.Write([]byte("ok"))
-}