Merge branch 'master' into release-1.11

Chart.yaml

@@ -1,7 +1,7 @@
 # apiVersion v2 is Helm 3
 apiVersion: v2
 name: airflow
-version: 1.11.4
+version: 1.11.5
 description: Helm chart to deploy the Astronomer Platform Airflow module
 icon: https://airflow.apache.org/docs/apache-airflow/stable/_images/pin_large.png
 keywords:

templates/_helpers.yaml

@@ -288,3 +288,34 @@ Create the name of the cleanup service account to use
 {{- define "astro.registry_secret" -}}
   {{- default (printf "%s-registry" .Release.Name) .Values.airflow.registry.secretName }}
 {{- end }}
+
+{{- define "sideccar_container_spec" -}}
+- name: {{ .Values.loggingSidecar.name }}
+  image: "{{ .Values.loggingSidecar.image }}"
+  env:
+  - name: VECTOR_CONFIG
+    value: /etc/sidecar-logging/vector-config.yaml
+  - name: SIDECAR_LOGS
+    value: /var/log/sidecar-logging-consumer
+  - name: COMPONENT
+    valueFrom:
+      fieldRef:
+        apiVersion: v1
+        fieldPath: metadata.labels['component']
+  - name: RELEASE
+    valueFrom:
+      fieldRef:
+        apiVersion: v1
+        fieldPath: metadata.labels['release']
+  - name: WORKSPACE
+    valueFrom:
+      fieldRef:
+        apiVersion: v1
+        fieldPath: metadata.labels['workspace']
+  volumeMounts:
+  - mountPath: /etc/sidecar-logging
+    name: config-volume
+  - name: sidecar-logging-consumer
+    mountPath: /var/log/sidecar-logging-consumer
+    readOnly: False
+{{- end }}

templates/dag-deploy/dag-server-statefulset.yaml

@@ -19,7 +19,6 @@ spec:
   {{- if and .Values.dagDeploy.persistence.enabled .Values.dagDeploy.persistence.persistentVolumeClaimRetentionPolicy }}
   persistentVolumeClaimRetentionPolicy: {{- toYaml .Values.dagDeploy.persistence.persistentVolumeClaimRetentionPolicy  | nindent 4 }}
   {{ end }}
-  serviceName: {{ include "airflow.fullname" . }}-dag-server
   replicas: 1
   selector:
     matchLabels:
@@ -49,7 +48,14 @@ spec:
         - name: dag-server
           image: "{{ .Values.dagDeploy.images.dagServer.repository }}:{{ .Values.dagDeploy.images.dagServer.tag }}"
           imagePullPolicy: {{ .Values.dagDeploy.images.dagServer.pullPolicy }}
+          {{- if .Values.loggingSidecar.enabled }}
+          command: ["bash"]
+          args:
+            - -c
+            - sanic dag_deploy.server.app -H 0.0.0.0 1> >( tee -a /var/log/sidecar-logging-consumer/out.log ) 2> >( tee -a /var/log/sidecar-logging-consumer/err.log >&2 )
+          {{ else }}
           command: ["sanic", "dag_deploy.server.app", "-H", "0.0.0.0"]
+          {{ end }}
           ports:
           - name: server
             containerPort: {{ .Values.dagDeploy.ports.dagServerHttp }}
@@ -81,6 +87,10 @@ spec:
           - name: data
             mountPath: /data
             readOnly: false
+          {{- if .Values.loggingSidecar.enabled }}
+          - mountPath: /var/log/sidecar-logging-consumer
+            name: sidecar-logging-consumer
+          {{ end }}
 {{- if .Values.authSidecar.enabled }}
         - name: auth-proxy
           image: "{{ .Values.authSidecar.repository }}:{{ .Values.authSidecar.tag }}"
@@ -114,14 +124,24 @@ spec:
             name: nginx-sidecar-conf
             subPath: nginx.conf
 {{- end }}
+{{- if .Values.loggingSidecar.enabled }}
+{{- include "sideccar_container_spec" . | nindent 8 }}
+{{- end }}
 {{- if .Values.dagDeploy.extraContainers }}
 {{- toYaml .Values.dagDeploy.extraContainers | nindent 8 }}
 {{- end }}
-{{- if .Values.authSidecar.enabled }}
       volumes:
+{{- if .Values.authSidecar.enabled }}
       - name: nginx-sidecar-conf
         configMap:
           name: {{ .Release.Name }}-dag-server-nginx-conf
+{{- end }}
+{{- if .Values.loggingSidecar.enabled }}
+      - name: config-volume
+        configMap:
+          name: {{ .Release.Name }}-sidecar-config
+      - emptyDir: {}
+        name: sidecar-logging-consumer
 {{- end }}
   volumeClaimTemplates:
   - metadata:

templates/flower/flower-auth-sidecar-configmap.yaml

@@ -23,7 +23,7 @@ data:
       }
     http {
       upstream astro-flower {
-        server localhost:{{ .Values.airflow.ports.flowerUI }} ;
+        server 127.0.0.1:{{ .Values.airflow.ports.flowerUI }} ;
       }
       server {
         server_name {{ .Release.Name }}-flower.{{ .Values.ingress.baseDomain }} ;

templates/git-sync-relay/git-sync-relay-deployment.yaml

@@ -127,34 +127,7 @@ spec:
           resources:
             {{- toYaml .Values.gitSyncRelay.gitDaemonResources | nindent 12 }}
         {{- if .Values.loggingSidecar.enabled }}
-        - name: vector
-          image: "{{ .Values.loggingSidecar.image }}"
-          env:
-          - name: VECTOR_CONFIG
-            value: /etc/sidecar-logging/vector-config.yaml
-          - name: SIDECAR_LOGS
-            value: /var/log/sidecar-logging-consumer
-          - name: COMPONENT
-            valueFrom:
-              fieldRef:
-                apiVersion: v1
-                fieldPath: metadata.labels['component']
-          - name: RELEASE
-            valueFrom:
-              fieldRef:
-                apiVersion: v1
-                fieldPath: metadata.labels['release']
-          - name: WORKSPACE
-            valueFrom:
-              fieldRef:
-                apiVersion: v1
-                fieldPath: metadata.labels['workspace']
-          volumeMounts:
-          - mountPath: /etc/sidecar-logging
-            name: config-volume
-          - name: sidecar-logging-consumer
-            mountPath: /var/log/sidecar-logging-consumer
-            readOnly: False
+{{- include "sideccar_container_spec" . | nindent 8 }}
           {{- end }}
 
 {{- if .Values.gitSyncRelay.extraContainers }}

templates/logging-sidecar-configmap.yaml

@@ -66,6 +66,22 @@ data:
           type: "vrl"
           source: 'includes(["git-sync-relay"], .component)'
 
+      filter_dagserver_logs:
+        type: filter
+        inputs:
+          - transform_airflow_logs
+        condition:
+          type: "vrl"
+          source: 'includes(["dag-server"], .component)'
+
+      filter_airflow_downgrade_logs:
+        type: filter
+        inputs:
+          - transform_airflow_logs
+        condition:
+          type: "vrl"
+          source: 'includes(["airflow-downgrade"], .component)'
+
       transform_task_log:
         type: remap
         inputs:
@@ -96,6 +112,8 @@ data:
           - final_task_log
           - filter_common_logs
           - filter_gitsyncrelay_logs
+          - filter_dagserver_logs
+          - filter_airflow_downgrade_logs
         source: |
           del(.host)
           del(.file)

templates/webserver/webserver-auth-sidecar-configmap.yaml

@@ -23,7 +23,7 @@ data:
       }
     http {
       upstream astro-webserver {
-        server localhost:{{ .Values.airflow.ports.airflowUI }} ;
+        server 127.0.0.1:{{ .Values.airflow.ports.airflowUI }} ;
       }
       server {
         server_name {{ .Release.Name }}.{{ .Values.ingress.baseDomain }} ;

tests/chart/test_dag_server_statefulset.py

@@ -192,3 +192,53 @@ def test_dag_server_statefulset_with_persistentVolumeClaimRetentionPolicy_overri
             persistentVolumeClaimRetentionPolicy
             == doc["spec"]["persistentVolumeClaimRetentionPolicy"]
         )
+
+    def test_dag_server_statefulset_with_sidecar_enabled(self, kube_version):
+        """Test dag-server statefulset with custom registry secret."""
+        values = {"dagDeploy": {"enabled": True}, "loggingSidecar": {"enabled": True}}
+
+        docs = render_chart(
+            kube_version=kube_version,
+            show_only="templates/dag-deploy/dag-server-statefulset.yaml",
+            values=values,
+        )
+        assert len(docs) == 1
+        doc = docs[0]
+
+        c_by_name = get_containers_by_name(doc)
+        assert len(c_by_name) == 2
+        assert c_by_name["dag-server"]["command"] == ["bash"]
+        c_by_name["dag-server"]["args"] == [
+            "-c",
+            "sanic dag_deploy.server.app -H 0.0.0.0 1> >( tee -a /var/log/sidecar-logging-consumer/out.log ) 2> >( tee -a /var/log/sidecar-logging-consumer/err.log >&2 )",
+        ]
+        assert "sidecar-log-consumer" in c_by_name
+
+    def test_dag_server_statefulset_with_sidecar_and_authproxy_enabled(
+        self, kube_version
+    ):
+        """Test dag-server statefulset with custom registry secret."""
+        values = {
+            "dagDeploy": {"enabled": True},
+            "loggingSidecar": {"enabled": True},
+            "authSidecar": {"enabled": True},
+        }
+
+        docs = render_chart(
+            kube_version=kube_version,
+            show_only="templates/dag-deploy/dag-server-statefulset.yaml",
+            values=values,
+        )
+        assert len(docs) == 1
+        doc = docs[0]
+
+        c_by_name = get_containers_by_name(doc)
+        assert len(c_by_name) == 3
+        assert "dag-server" in c_by_name
+        assert "auth-proxy" in c_by_name
+        assert c_by_name["dag-server"]["command"] == ["bash"]
+        c_by_name["dag-server"]["args"] == [
+            "-c",
+            "sanic dag_deploy.server.app -H 0.0.0.0 1> >( tee -a /var/log/sidecar-logging-consumer/out.log ) 2> >( tee -a /var/log/sidecar-logging-consumer/err.log >&2 )",
+        ]
+        assert "sidecar-log-consumer" in c_by_name

values.yaml

@@ -32,13 +32,13 @@ airflow:
       tag: 0.26.1
     redis:
       repository: quay.io/astronomer/ap-redis
-      tag: 6.2.14
+      tag: 7.2.5
     pgbouncer:
       repository: quay.io/astronomer/ap-pgbouncer
-      tag: 1.21.0
+      tag: 1.23.1-1
     pgbouncerExporter:
       repository: quay.io/astronomer/ap-pgbouncer-exporter
-      tag: 0.15.0-9
+      tag: 0.15.0-10
     gitSync:
       repository: quay.io/astronomer/ap-git-sync
       tag: 3.6.9
@@ -483,7 +483,7 @@ authSidecar:
 
 loggingSidecar:
   enabled: false
-  name: sidecar-logging-consumer
+  name: sidecar-log-consumer
   customConfig: false
   indexPattern: "%Y.%m.%d"
   indexNamePrefix: vector
@@ -543,7 +543,7 @@ dagDeploy:
   images:
     dagServer:
       repository: quay.io/astronomer/ap-dag-deploy
-      tag: 0.5.0
+      tag: 0.5.1
       imagePullPolicy: IfNotPresent
   livenessProbe:
     initialDelaySeconds: 30
@@ -589,10 +589,10 @@ gitSyncRelay:
   images:
     gitDaemon:
       repository: "quay.io/astronomer/ap-git-daemon"
-      tag: "3.18.7-1"
+      tag: "3.18.8"
     gitSync:
       repository: "quay.io/astronomer/ap-git-sync-relay"
-      tag: "0.0.3-4"
+      tag: "0.0.3-5"
   repo:
     url: ~
     branch: main