Merge branch 'release/0.55.0' into main

airyhq · May 9, 2024 · 418ac5c · 418ac5c
2 parents a478a38 + 397e04b
commit 418ac5c
Show file tree

Hide file tree

Showing 137 changed files with 4,455 additions and 316 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -54,6 +54,16 @@ jobs:
         run: |
           bazel test --test_tag_filters=-lint //...
 
+      - name: Cleanup space
+        run: |
+          df -h
+          sudo apt-get autoremove -y
+          sudo apt-get clean
+          docker images prune -a
+          sudo rm -rf /usr/local/share/powershell
+          sudo rm -rf /opt/hostedtoolcache
+          df -h
+
       - name: Build all artifacts
         run: |
           bazel build //...
@@ -64,18 +74,14 @@ jobs:
           echo ${{ secrets.PAT }} | docker login ghcr.io -u airydevci --password-stdin
           ./scripts/push-images.sh
 
-      - name: Install aws cli
-        uses: chrislennon/[email protected]
-        env:
-          ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true'
-
       - name: Upload airy binary to S3
         if: ${{ github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/release') || github.ref == 'refs/heads/main' }}
         run: |
           ./scripts/upload-cli-binaries.sh
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: ${{ secrets.AWS_REGION }}
           GITHUB_BRANCH: ${{ github.ref }}
 
       - name: Publish helm charts

diff --git a/README.md b/README.md
@@ -15,7 +15,7 @@
 
 ---
 
-![Airy_Explainer_Highlevel_Readme](https://airy.co/docs/core/img/getting-started/introduction-light.png)
+![Airy_Explainer_Highlevel_Readme](https://airy.co/docs/core/img/getting-started/introduction.png)
 
 Airy Core is an is an open-source streaming app framework to train ML models and supply them with historical and real-time data. With Airy you can process data from a variety of
 sources:

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-0.54.0
+0.55.0
diff --git a/backend/components/chat-plugin/helm/templates/backend/deployment.yaml b/backend/components/chat-plugin/helm/templates/backend/deployment.yaml
@@ -51,6 +51,11 @@ spec:
           initialDelaySeconds: 120
           periodSeconds: 10
           failureThreshold: 3
+{{ if .Values.global.kafkaCertAuth }}
+        volumeMounts:
+        - name: kafka-config-certs
+          mountPath: /opt/kafka/certs
+{{ end }}
         resources:
 {{ toYaml .Values.backend.resources | indent 10 }}
       initContainers:
@@ -68,3 +73,8 @@ spec:
         - name: provisioning-scripts
           configMap:
             name: provisioning-scripts
+{{ if .Values.global.kafkaCertAuth }}
+        - name: kafka-config-certs
+          configMap:
+            name: kafka-config-certs
+{{ end }}
diff --git a/backend/components/contacts/helm/templates/deployment.yaml b/backend/components/contacts/helm/templates/deployment.yaml
@@ -45,6 +45,11 @@ spec:
           initialDelaySeconds: 120
           periodSeconds: 10
           failureThreshold: 3
+{{ if .Values.global.kafkaCertAuth }}
+        volumeMounts:
+        - name: kafka-config-certs
+          mountPath: /opt/kafka/certs
+{{ end }}
         resources:
 {{ toYaml .Values.resources | indent 10 }}
       initContainers:
@@ -62,3 +67,8 @@ spec:
         - name: provisioning-scripts
           configMap:
             name: provisioning-scripts
+{{ if .Values.global.kafkaCertAuth }}
+        - name: kafka-config-certs
+          configMap:
+            name: kafka-config-certs
+{{ end }}
diff --git a/backend/components/facebook/helm/templates/deployments.yaml b/backend/components/facebook/helm/templates/deployments.yaml
@@ -59,6 +59,11 @@ spec:
               - name: Health-Check
                 value: health-check
             initialDelaySeconds: 120
+{{ if .Values.global.kafkaCertAuth }}
+          volumeMounts:
+          - name: kafka-config-certs
+            mountPath: /opt/kafka/certs
+{{ end }}
           resources:
 {{ toYaml .Values.connector.resources | indent 12 }}
       initContainers:
@@ -76,6 +81,11 @@ spec:
         - name: provisioning-scripts
           configMap:
             name: provisioning-scripts
+{{ if .Values.global.kafkaCertAuth }}
+        - name: kafka-config-certs
+          configMap:
+            name: kafka-config-certs
+{{ end }}
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -124,6 +134,11 @@ spec:
           initialDelaySeconds: 120
           periodSeconds: 10
           failureThreshold: 3
+{{ if .Values.global.kafkaCertAuth }}
+        volumeMounts:
+        - name: kafka-config-certs
+          mountPath: /opt/kafka/certs
+{{ end }}
         resources:
 {{ toYaml .Values.eventsRouter.resources | indent 10 }}
       initContainers:
@@ -141,3 +156,8 @@ spec:
         - name: provisioning-scripts
           configMap:
             name: provisioning-scripts
+{{ if .Values.global.kafkaCertAuth }}
+        - name: kafka-config-certs
+          configMap:
+            name: kafka-config-certs
+{{ end }}
diff --git a/backend/components/flink-connector/Dockerfile.result-sender b/backend/components/flink-connector/Dockerfile.result-sender
@@ -0,0 +1,16 @@
+FROM golang:1.17
+
+WORKDIR /app
+
+COPY ./src/types.go ./src/tools.go ./src/result-sender.go ./
+
+RUN go mod init main && \
+    go get github.com/confluentinc/confluent-kafka-go/v2/kafka && \
+    go get github.com/confluentinc/confluent-kafka-go/v2/schemaregistry && \
+    go get github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde && \
+    go get github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde/avro && \
+    go get golang.org/x/net
+
+RUN go build -o app
+
+CMD ["./app"]
diff --git a/backend/components/flink-connector/Dockerfile.statements-executor b/backend/components/flink-connector/Dockerfile.statements-executor
@@ -0,0 +1,16 @@
+FROM golang:1.17
+
+WORKDIR /app
+
+COPY ./src/types.go ./src/tools.go ./src/statements-executor.go ./
+
+RUN go mod init main && \
+    go get github.com/confluentinc/confluent-kafka-go/v2/kafka && \
+    go get github.com/confluentinc/confluent-kafka-go/v2/schemaregistry && \
+    go get github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde && \
+    go get github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde/avro && \
+    go get golang.org/x/net
+
+RUN go build -o app
+
+CMD ["./app"]
diff --git a/backend/components/flink-connector/Makefile b/backend/components/flink-connector/Makefile
@@ -0,0 +1,13 @@
+build-statements-executor:
+	docker build -t flink-connector/statements-executor -f Dockerfile.statements-executor .
+
+release-statements-executor: build-statements-executor
+	docker tag flink-connector/statements-executor ghcr.io/airyhq/connectors/flink/statements-executor:release
+	docker push ghcr.io/airyhq/connectors/flink/statements-executor:release
+
+build-result-sender:
+	docker build -t flink-connector/result-sender -f Dockerfile.result-sender .
+
+release-result-sender: build-result-sender
+	docker tag flink-connector/result-sender ghcr.io/airyhq/connectors/flink/result-sender:release
+	docker push ghcr.io/airyhq/connectors/flink/result-sender:release
diff --git a/backend/components/flink-connector/helm/BUILD b/backend/components/flink-connector/helm/BUILD
@@ -0,0 +1,3 @@
+load("//tools/build:helm.bzl", "helm_ruleset_core_version")
+
+helm_ruleset_core_version()
diff --git a/backend/components/flink-connector/helm/Chart.yaml b/backend/components/flink-connector/helm/Chart.yaml
@@ -0,0 +1,6 @@
+
+apiVersion: v2
+appVersion: "1.0"
+description: Flink connector
+name: flink-connector
+version: 1.0
diff --git a/backend/components/flink-connector/helm/templates/configmap.yaml b/backend/components/flink-connector/helm/templates/configmap.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.component }}
+  labels:
+    core.airy.co/managed: "true"
+    core.airy.co/mandatory: "{{ .Values.mandatory }}"
+    core.airy.co/component: "{{ .Values.component }}"
+  annotations:
+    core.airy.co/enabled: "{{ .Values.enabled }}"
diff --git a/backend/components/flink-connector/helm/templates/result-sender/deployment.yaml b/backend/components/flink-connector/helm/templates/result-sender/deployment.yaml
@@ -0,0 +1,50 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Values.component }}-{{ .Values.resultSender.name }}
+  labels:
+    app: {{ .Values.component }}
+    core.airy.co/managed: "true"
+    core.airy.co/mandatory: "{{ .Values.mandatory }}"
+    core.airy.co/component: {{ .Values.component }}
+spec:
+  replicas: {{ if .Values.enabled }} 1 {{ else }} 0 {{ end }}
+  selector:
+    matchLabels:
+      app: {{ .Values.component }}-{{ .Values.resultSender.name }}
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.component }}-{{ .Values.resultSender.name }}
+    spec:
+      containers:
+        - name: app
+          image: "ghcr.io/airyhq/{{ .Values.resultSender.image }}:release"
+          imagePullPolicy: Always
+          envFrom:
+            - configMapRef:
+                name: security
+            - configMapRef:
+                name: kafka-config
+            - configMapRef:
+                name: {{ .Values.component }}
+          env:
+            - name: KAFKA_TOPIC_NAME
+              value: {{ .Values.resultSender.topic }}
+            - name: API_COMMUNICATION_URL
+              value: {{ .Values.apiCommunicationUrl }}
+          livenessProbe:
+            httpGet:
+              path: /actuator/health
+              port: {{ .Values.port }}
+              httpHeaders:
+                - name: Health-Check
+                  value: health-check
+            initialDelaySeconds: 43200
+            periodSeconds: 10
+            failureThreshold: 3
diff --git a/backend/components/flink-connector/helm/templates/result-sender/service.yaml b/backend/components/flink-connector/helm/templates/result-sender/service.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.component }}-{{ .Values.resultSender.name }}
+  labels:
+    app: {{ .Values.component }}-{{ .Values.resultSender.name }}
+spec:
+  type: ClusterIP
+  clusterIP: None
+  ports:
+    - name: {{ .Values.component }}-{{ .Values.resultSender.name }}
+      port: 80
+      targetPort: {{ .Values.port }}
+  selector:
+    app: {{ .Values.component }}-{{ .Values.resultSender.name }}
diff --git a/backend/components/flink-connector/helm/templates/statements-executor/deployment.yaml b/backend/components/flink-connector/helm/templates/statements-executor/deployment.yaml
@@ -0,0 +1,50 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Values.component }}-{{ .Values.executor.name }}
+  labels:
+    app: {{ .Values.component }}
+    core.airy.co/managed: "true"
+    core.airy.co/mandatory: "{{ .Values.mandatory }}"
+    core.airy.co/component: {{ .Values.component }}
+spec:
+  replicas: {{ if .Values.enabled }} 1 {{ else }} 0 {{ end }}
+  selector:
+    matchLabels:
+      app: {{ .Values.component }}-{{ .Values.executor.name }}
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.component }}-{{ .Values.executor.name }}
+    spec:
+      containers:
+        - name: app
+          image: "ghcr.io/airyhq/{{ .Values.executor.image }}:release"
+          imagePullPolicy: Always
+          envFrom:
+            - configMapRef:
+                name: security
+            - configMapRef:
+                name: kafka-config
+            - configMapRef:
+                name: {{ .Values.component }}
+          env:
+            - name: KAFKA_TOPIC_NAME
+              value: {{ .Values.executor.topic }}
+            - name: FLINK_GATEWAY_URL
+              value: {{ .Values.gatewayUrl }}
+          livenessProbe:
+            httpGet:
+              path: /actuator/health
+              port: {{ .Values.port }}
+              httpHeaders:
+                - name: Health-Check
+                  value: health-check
+            initialDelaySeconds: 43200
+            periodSeconds: 10
+            failureThreshold: 3
diff --git a/backend/components/flink-connector/helm/templates/statements-executor/service.yaml b/backend/components/flink-connector/helm/templates/statements-executor/service.yaml
@@ -0,0 +1,16 @@
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.component }}-{{ .Values.executor.name }}
+  labels:
+    app: {{ .Values.component }}-{{ .Values.executor.name }}
+spec:
+  type: ClusterIP
+  clusterIP: None
+  ports:
+    - name: {{ .Values.component }}-{{ .Values.executor.name }}
+      port: 80
+      targetPort: {{ .Values.port }}
+  selector:
+    app: {{ .Values.component }}-{{ .Values.executor.name }}
diff --git a/backend/components/flink-connector/helm/values.yaml b/backend/components/flink-connector/helm/values.yaml
@@ -0,0 +1,16 @@
+
+component: flink-connector
+mandatory: false
+enabled: false
+port: 8080
+resources:
+gatewayUrl: "http://flink-jobmanager:8083"
+apiCommunicationUrl: "http://api-communication/messages.send"
+executor:
+  name: statements-executor
+  image: connectors/flink/statements-executor
+  topic: flink.statements
+resultSender:
+  name: result-sender
+  image: connectors/flink/result-sender
+  topic: flink.output