Merge branch 'master' of github.com:dmwm/CMSKubernetes

docker/auth-proxy-server/Dockerfile

@@ -5,7 +5,7 @@ ENV USER=http
 WORKDIR $WDIR
 
 # tag to use
-ENV TAG=0.2.53
+ENV TAG=0.2.81
 
 ARG CGO_ENABLED=0
 RUN mkdir $WDIR/gopath
@@ -14,12 +14,9 @@ RUN git clone https://github.com/vkuznet/auth-proxy-server.git
 WORKDIR $WDIR/auth-proxy-server
 RUN git checkout tags/$TAG -b build && make
 
-FROM cmssw/cmsweb-base:latest as cmsweb-base
-
 # https://blog.baeke.info/2021/03/28/distroless-or-scratch-for-go-apps/
 # FROM alpine
 FROM gcr.io/distroless/static AS final
 #RUN mkdir -p /data/static && mkdir -p /data/srv/logs/frontend
 COPY --from=go-builder /data/auth-proxy-server/auth-proxy-server /data/
 COPY --from=go-builder /data/auth-proxy-server/static/cmsmon/index.html /data/static/
-COPY --from=cmsweb-base /etc/grid-security /etc/grid-security

docker/httpgo/httpgo.go

@@ -5,6 +5,8 @@ import (
 	"log"
 	"net/http"
 	"net/http/httputil"
+	"sort"
+	"strings"
 )
 
 func RequestHandler(w http.ResponseWriter, r *http.Request) {
@@ -13,9 +15,17 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) {
 		fmt.Println("TLS:", r.TLS)
 		fmt.Println("Header:", r.Header)
 
+		// get all HTTP headers and sort them out
+		var headers []string
+		for k, _ := range r.Header {
+			headers = append(headers, k)
+		}
+		sort.Strings(headers)
+
 		// print out all request headers
 		fmt.Fprintf(w, "%s %s %s \n", r.Method, r.URL, r.Proto)
-		for k, v := range r.Header {
+		for _, k := range headers {
+			v, _ := r.Header[k]
 			h := strings.ToLower(k)
 			if strings.Contains(h, "hmac") || strings.Contains(h, "cookie") {
 				continue

docker/pypi/dmwm-base/manage

@@ -36,6 +36,10 @@ export X509_USER_KEY=$AUTHDIR/dmwm-service-key.pem
 export X509_USER_CERT=$AUTHDIR/dmwm-service-cert.pem
 export REQMGR_CACHE_DIR=$STATEDIR
 export WMCORE_CACHE_DIR=$STATEDIR
+# MSUnmerged also needs to access a proxy with additional voms roles
+if [ -f $AUTHDIR/proxy.cert ]; then
+    export X509_USER_PROXY=$AUTHDIR/proxy.cert
+fi
 
 # by default Rucio relies on /opt/rucio/etc/config.cfg
 # if necessary we may setup RUCIO_HOME which should provide this location

docker/pypi/wmagent/wmagent-docker-run.sh

@@ -135,3 +135,6 @@ echo "Checking if there is no other wmagent container running and creating a lin
 echo "Starting wmagent:$WMA_TAG docker container with user: $wmaUser:$wmaGroup"
 docker run $dockerOpts $registry/$repository:$WMA_TAG
 docker exec -u root -it wmagent service cron start
+
+# Workaround su authentication issue (cron uses setuid via su)
+docker exec -u root -it wmagent sh -c "echo $wmaUser:$wmaUser | chpasswd"

helm/crabserver/Chart.yaml

@@ -15,9 +15,9 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.10
+version: 0.1.11
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: "v3.240530"
+appVersion: "v3.240731"

helm/crabserver/README.md

@@ -8,26 +8,12 @@ for some context.
 We like to use helm for templating, using the helm charts to generate the 
 manifest file, then applying it with `kubectl apply`.
 
-We therefore ask cmsweb operators to deploy crab with the following procedure:
+We therefore ask cmsweb operators to deploy crab following the procedure at
+[1].
 
-Generate the manifest with
-
-```bash
-# testbed
-helm template crabserver . -f values.yaml -f values-testbed.yaml > ../../kubernetes/cmsweb/services/crabserver.yaml
-
-# prod
-helm template crabserver . -f values.yaml -f values-prod.yaml > ../../kubernetes/cmsweb/services/crabserver.yaml
-```
-
-Then to deploy it with the usual `deploy-srv.sh` script
+Changes to `../../kubernetes/cmsweb/services/crabserver.yaml` should not be committed.
 
-```bash
-# testbed
-./scripts/deploy-srv.sh crabserver v3.231006 preprod
+---
 
-# prod
-./scripts/deploy-srv.sh crabserver v3.231006 prod
-```
+[1] https://cmscrab.docs.cern.ch/technical/crab-rest/deploy.html#deploy-on-kubernetes-use-helm-template-to-generate-manifest-preferred
 
-Changes to `../../kubernetes/cmsweb/services/crabserver.yaml` should not be committed.

helm/crabserver/deploy.sh

@@ -1,17 +1,29 @@
 #! /bin/bash
 set -euo pipefail
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+
+# hash table of clusters nickname-name:
+declare -A cluster_map=([prod]=prod)
+cluster_map[prod]=k8s-prodsrv
+cluster_map[preprod]=k8s-prodsrv-v1.22.9
+cluster_map[testbed]=testbed
+cluster_map[test2]=test2
+cluster_map[test11]=test11
+cluster_map[test12]=test12
 if [[ $# -ne 1 ]]; then
     echo "Usage: deploy.sh ENVNAME"
-    echo " ENVNAME=(prod|testbed|test2|test11|test12)"
+    echo " ENVNAME=(prod|preprod|testbed|test2|test11|test12)"
     exit 1
 fi
-desired_cluster=$1
+desired_cluster="${cluster_map[$1]}"
+
 # make sure that your current context points to the desired cluster
 current_cluster=$(kubectl config view -o json | jq '.["current-context"] as $context | .["contexts"][] | select(.name | contains($context))| .context.cluster')
+
 if [[ $current_cluster =~ $desired_cluster ]]; then 
   echo "deploying to $desired_cluster"; 
-  helm template crabserver . -f values.yaml -f values-$desired_cluster.yaml | kubectl -n crab apply -f -
+  helm template crabserver . -f values.yaml -f values-${1}-pypi.yaml | kubectl -n crab apply -f -
 else 
   echo "wrong cluster: your are connected to $current_cluster"; 
 fi

helm/crabserver/values-preprod-pypi.yaml

@@ -0,0 +1,18 @@
+---
+environment: "preprod"
+
+image:
+  path: registry.cern.ch/cmscrab/crabserver
+  pullPolicy: IfNotPresent
+  tag: "v3.240731-stable"
+  command:
+  - /data/entrypoint.sh
+  args:
+  - /bin/bash
+  - -c
+  - |
+    sudo cp -r /host/etc/grid-security/* /etc/grid-security \
+    && echo 'INFO Files in /etc/grid-security' \
+    && ls -lahZ /etc/grid-security \
+    && /data/run.sh
+

helm/crabserver/values-prod-pypi.yaml

@@ -0,0 +1,18 @@
+---
+environment: "prod"
+
+image:
+  path: registry.cern.ch/cmscrab/crabserver
+  pullPolicy: IfNotPresent
+  tag: "v3.240731-stable"
+  command:
+  - /data/entrypoint.sh
+  args:
+  - /bin/bash
+  - -c
+  - |
+    sudo cp -r /host/etc/grid-security/* /etc/grid-security \
+    && echo 'INFO Files in /etc/grid-security' \
+    && ls -lahZ /etc/grid-security \
+    && /data/run.sh
+

helm/crabserver/values-prod.yaml

@@ -1,5 +1,5 @@
 environment: "prod"
 
 image:
-  tag: "v3.240508"
-
+  tag: "v3.240731"
+  path: "registry.cern.ch/cmsweb/crabserver"

helm/crabserver/values-test1.yaml

@@ -0,0 +1,4 @@
+environment: "test"
+
+image:
+  tag: "v3.240709"

helm/crabserver/values-test11-pypi.yaml

@@ -11,7 +11,7 @@ image:
   - /bin/bash
   - -c
   - |
-    sudo cp /host/etc/grid-security/* /etc/grid-security \
+    sudo cp -r /host/etc/grid-security/* /etc/grid-security \
     && echo 'INFO Files in /etc/grid-security' \
     && ls -lahZ /etc/grid-security \
     && /data/run.sh

helm/crabserver/values-testbed-pypi.yaml

@@ -0,0 +1,18 @@
+---
+environment: "testbed"
+
+image:
+  path: registry.cern.ch/cmscrab/crabserver
+  pullPolicy: IfNotPresent
+  tag: "v3.240731-stable"
+  command:
+  - /data/entrypoint.sh
+  args:
+  - /bin/bash
+  - -c
+  - |
+    sudo cp -r /host/etc/grid-security/* /etc/grid-security \
+    && echo 'INFO Files in /etc/grid-security' \
+    && ls -lahZ /etc/grid-security \
+    && /data/run.sh
+

helm/crabserver/values.yaml

@@ -8,7 +8,7 @@ replicaCount:
   test: 1
 
 image:
-  path: registry.cern.ch/cmsweb/crabserver
+  path: registry.cern.ch/cmscrab/crabserver
   pullPolicy: Always
   # Overrides the image tag whose default is the chart appVersion.
   tag: "v3.231006"

kubernetes/cmsweb/daemonset/auth-proxy-server.yaml

@@ -18,10 +18,12 @@ data:
       backoff: 5s
       max_backoff: 10s
       tags: ["aps"]
-    output.console:
-      codec.format:
-        string: '%{[message]} - Podname=${MY_POD_NAME}'
-        pretty: false
+    output.logstash:
+      hosts: ["logstash.monitoring:5044"]
+      compression_level: 3
+      worker: 4
+      bulk_max_size: 4096
+      pipelining: 2
     queue.mem:
       events: 65536
     logging.metrics.enabled: false
@@ -103,7 +105,7 @@ spec:
         - name: token-secrets
           mountPath: /etc/token
           readOnly: true   
-        - mountPath: /etc/grid-security/certificates
+        - mountPath: /host/etc/grid-security
           name: etc-grid-security
           readOnly: true   
         - name: hostcert
@@ -165,7 +167,7 @@ spec:
           secretName: token-secrets
       - name: etc-grid-security
         hostPath:
-            path: /etc/grid-security/certificates
+            path: /etc/grid-security
       - name: hostcert
         hostPath:
             path: /etc/grid-security/hostcert.pem
@@ -189,7 +191,7 @@ spec:
 #PROD#      claimName: filebeat-cephfs-claim-default
 #PROD#- name: logs
 #PROD#  persistentVolumeClaim:
-#PROD#      claimName: logs-cephfs-claim-aps
+#PROD#      claimName: logs-cephfs-claim-default
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

kubernetes/cmsweb/daemonset/x509-proxy-server.yaml

@@ -17,10 +17,13 @@ data:
       scan_frequency: 10s
       backoff: 5s
       max_backoff: 10s
-    output.console:
-      codec.format:
-        string: '%{[message]} - Podname=${MY_POD_NAME}'
-        pretty: false
+      tags: ["aps"]
+    output.logstash:
+      hosts: ["logstash.monitoring:5044"]
+      compression_level: 3
+      worker: 4
+      bulk_max_size: 4096
+      pipelining: 2
     queue.mem:
       events: 65536
     logging.metrics.enabled: false
@@ -112,7 +115,7 @@ spec:
         - name: token-secrets
           mountPath: /etc/token
           readOnly: true 
-        - mountPath: /etc/grid-security/certificates
+        - mountPath: /host/etc/grid-security
           name: etc-grid-security
           readOnly: true
 #PROD#  - name: logs
@@ -176,7 +179,7 @@ spec:
           secretName: token-secrets    
       - name: etc-grid-security
         hostPath:
-            path: /etc/grid-security/certificates
+            path: /etc/grid-security
 #PROD#- name: varlog
 #PROD#  hostPath:
 #PROD#    path: /var/log
@@ -194,7 +197,7 @@ spec:
 #PROD#      claimName: filebeat-cephfs-claim-default
 #PROD#- name: logs
 #PROD#  persistentVolumeClaim:
-#PROD#      claimName: logs-cephfs-claim-aps
+#PROD#      claimName: logs-cephfs-claim-default
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

kubernetes/cmsweb/monitoring/logstash.conf

@@ -58,7 +58,6 @@ filter {
       }
       if ![api] {
           mutate { replace => { "api" => "%{request}" } }
-          mutate { replace => { "system" => "%{request}" } }
       }
       if [client] {
           grok { match => { "client" => '%{DATA:client_name}/%{DATA:client_version}$' } }
@@ -79,12 +78,16 @@ filter {
   if "aps" in [tags] {
       mutate { replace => { "type" => "aps" } }
       grok {
-        match => { "message" => '\[%{TIMESTAMP_ISO8601:tstamp}\] %{DATA:httpversion} %{NUMBER:code:int} %{WORD:method} %{NOTSPACE:request} \[data: %{NUMBER:bytes_received:int} in %{NUMBER:bytes_sent:int} out\] \[host: %{IPORHOST:frontend}:%{NUMBER:fe_port}\] \[remoteAddr: %{IPORHOST:clientip}:%{NUMBER:clientport:int}\] \[X-Forwarded-For: %{IPORHOST:x_forwarded_ip}:%{NUMBER:x_forwarded_port:int}\] \[X-Forwarded-Host: %{HOSTNAME:x_forwarded_host}\] \[auth: %{DATA:tls} %{DATA:crypto} "%{DATA:dn}" %{DATA:auth_name} %{WORD:auth_protocol}\] \[ref: "%{DATA:cluster}" "%{DATA:client}"\] \[req: %{NUMBER:request_time:float} \(s\) proxy-resp: %{NUMBER:proxy_resp_time:float} \(s\)\]' }
+        match => { "message" => '\[%{TIMESTAMP_ISO8601:tstamp}\] %{DATA:httpversion} %{NUMBER:code:int} %{WORD:method} %{NOTSPACE:request} \[data: %{NUMBER:bytes_received:int} in %{NUMBER:bytes_sent:int} out\] \[host: %{IPORHOST:frontend}(?::%{NUMBER:fe_port})?\] \[remoteAddr: %{IPORHOST:clientip}:%{NUMBER:clientport:int}\] \[X-Forwarded-For: (%{IPORHOST:x_forwarded_ip}:%{NUMBER:x_forwarded_port:int})?\] \[X-Forwarded-Host: (%{HOSTNAME:x_forwarded_host})?\] \[auth: %{DATA:tls} %{DATA:crypto} "%{DATA:dn}" %{DATA:auth_name} %{WORD:auth_protocol}\] \[ref: "%{DATA:cluster}" "%{DATA:client}"\] \[req: %{NUMBER:request_time:float} \(s\) proxy-resp: %{NUMBER:proxy_resp_time:float} \(s\)\]' }
+      }
+      grok {
+        match => {
+          "cmsweb_log" => "/data/srv/logs/frontend/%{DATA:log_prefix}_%{GREEDYDATA:proxy_server}-%{DATA}_%{YEAR}%{MONTHNUM}%{MONTHDAY}"
+        }
       }
-
       grok {
-      pattern_definitions => { "WORDHYPHEN" => "\b[\w\-]+\b" }
-      match => { "request" => '/%{WORDHYPHEN:system}%{UNIXPATH:uri_path}%{URIPARAM:uri_params}?' }
+        pattern_definitions => { "WORDHYPHEN" => "\b[\w\-]+\b" }
+        match => { "request" => '/%{WORDHYPHEN:system}%{UNIXPATH:uri_path}%{URIPARAM:uri_params}?' }
       }
       if [system] =~ /^(wmstatsserver|reqmgr2|t0_reqmon|ms-pileup|ms-transferor|ms-monitor|ms-output|ms-unmerged|ms-rulecleaner)$/ {
           grok {
@@ -121,7 +124,6 @@ filter {
       }
       if ![api] {
           mutate { replace => { "api" => "%{request}" } }
-          mutate { replace => { "system" => "%{request}" } }
       }
       if [client] {
           grok { match => { "client" => '%{DATA:client_name}/%{DATA:client_version}$' } }
@@ -237,8 +239,6 @@ filter {
 
   # common filters
 
-  # drop failed records
-  if "_grokparsefailure" in [tags] { drop { } }
   # remove quotes from message entry since it will break the JSON
   mutate { gsub => [ "message", "\n", "", "message", "\"", ""] }