From 05c5f8a982fbd80a956e1ef01145f3e1770f0363 Mon Sep 17 00:00:00 2001 From: ljupcovangelski Date: Thu, 4 Apr 2024 10:55:19 +0200 Subject: [PATCH 1/4] Fixes #4156 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index b93005464..316ba4bd9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.55.0-alpha +0.55.0 From 2f052cd199c6df9cca03b738b7ddded0a0dfd59e Mon Sep 17 00:00:00 2001 From: ljupcovangelski Date: Wed, 10 Apr 2024 09:49:39 +0200 Subject: [PATCH 2/4] [#4156] Fix CI --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 02c8436e4..699da3a78 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -81,6 +81,7 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} GITHUB_BRANCH: ${{ github.ref }} - name: Publish helm charts From 2efde819c7ce4b762f4e2f9bf7c7071f17250ba4 Mon Sep 17 00:00:00 2001 From: Ljupco Vangelski Date: Mon, 6 May 2024 13:25:24 +0200 Subject: [PATCH 3/4] [#4167] Kafka certificate authentication (#4168) --- .../helm/templates/backend/deployment.yaml | 10 ++++++ .../contacts/helm/templates/deployment.yaml | 10 ++++++ .../facebook/helm/templates/deployments.yaml | 20 +++++++++++ .../google/helm/templates/deployments.yaml | 20 +++++++++++ .../helm/templates/deployment.yaml | 10 ++++++ .../helm/templates/deployment.yaml | 10 ++++++ .../streams/helm/templates/deployment.yaml | 10 ++++++ .../twilio/helm/templates/deployments.yaml | 10 ++++++ .../viber/helm/templates/deployments.yaml | 10 ++++++ .../webhook/helm/templates/deployments.yaml | 10 ++++++ .../whatsapp/helm/templates/deployments.yaml | 10 ++++++ .../docs/getting-started/installation/helm.md | 33 +++++++++++++++++++ .../components/api-admin/deployment.yaml | 10 ++++++ .../api-communication/deployment.yaml | 10 ++++++ .../api-components-installer/deployment.yaml | 11 ++++++- .../components/api-websocket/deployment.yaml | 10 ++++++ .../components/unread-counter/deployment.yaml | 10 ++++++ .../helm-chart/templates/config/kafka.yaml | 1 + infrastructure/helm-chart/values.yaml | 1 + .../airy/kafka/core/KafkaConsumerWrapper.java | 16 ++++++++- .../kafka/streams/KafkaStreamsWrapper.java | 19 ++++++++++- .../spring/kafka/core/KafkaCoreConfig.java | 21 ++++++++++-- .../kafka/streams/KafkaStreamsConfig.java | 5 ++- 23 files changed, 270 insertions(+), 7 deletions(-) diff --git a/backend/components/chat-plugin/helm/templates/backend/deployment.yaml b/backend/components/chat-plugin/helm/templates/backend/deployment.yaml index 1f8995b81..a82f25fd2 100644 --- a/backend/components/chat-plugin/helm/templates/backend/deployment.yaml +++ b/backend/components/chat-plugin/helm/templates/backend/deployment.yaml @@ -51,6 +51,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.backend.resources | indent 10 }} initContainers: @@ -68,3 +73,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/contacts/helm/templates/deployment.yaml b/backend/components/contacts/helm/templates/deployment.yaml index 5e4c6846a..b0d2e68d9 100644 --- a/backend/components/contacts/helm/templates/deployment.yaml +++ b/backend/components/contacts/helm/templates/deployment.yaml @@ -45,6 +45,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.resources | indent 10 }} initContainers: @@ -62,3 +67,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/facebook/helm/templates/deployments.yaml b/backend/components/facebook/helm/templates/deployments.yaml index 4d72ef517..a530a8122 100644 --- a/backend/components/facebook/helm/templates/deployments.yaml +++ b/backend/components/facebook/helm/templates/deployments.yaml @@ -59,6 +59,11 @@ spec: - name: Health-Check value: health-check initialDelaySeconds: 120 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.connector.resources | indent 12 }} initContainers: @@ -76,6 +81,11 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} --- apiVersion: apps/v1 kind: Deployment @@ -124,6 +134,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.eventsRouter.resources | indent 10 }} initContainers: @@ -141,3 +156,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/google/helm/templates/deployments.yaml b/backend/components/google/helm/templates/deployments.yaml index 7af611787..80c2af237 100644 --- a/backend/components/google/helm/templates/deployments.yaml +++ b/backend/components/google/helm/templates/deployments.yaml @@ -54,6 +54,11 @@ spec: - name: Health-Check value: health-check initialDelaySeconds: 120 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.connector.resources | indent 12 }} initContainers: @@ -71,6 +76,11 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} --- apiVersion: apps/v1 kind: Deployment @@ -122,6 +132,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.eventsRouter.resources | indent 10 }} initContainers: @@ -139,3 +154,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/media-resolver/helm/templates/deployment.yaml b/backend/components/media-resolver/helm/templates/deployment.yaml index b31af97ff..3484e0b71 100644 --- a/backend/components/media-resolver/helm/templates/deployment.yaml +++ b/backend/components/media-resolver/helm/templates/deployment.yaml @@ -45,6 +45,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.resources | indent 12 }} initContainers: @@ -62,3 +67,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/sources-api/helm/templates/deployment.yaml b/backend/components/sources-api/helm/templates/deployment.yaml index 9ce7cda66..4d23b16de 100644 --- a/backend/components/sources-api/helm/templates/deployment.yaml +++ b/backend/components/sources-api/helm/templates/deployment.yaml @@ -50,6 +50,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.resources | indent 10 }} initContainers: @@ -67,3 +72,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/streams/helm/templates/deployment.yaml b/backend/components/streams/helm/templates/deployment.yaml index 067e9bf6c..b0cd1dfb3 100644 --- a/backend/components/streams/helm/templates/deployment.yaml +++ b/backend/components/streams/helm/templates/deployment.yaml @@ -48,6 +48,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.resources | indent 10 }} initContainers: @@ -67,3 +72,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/twilio/helm/templates/deployments.yaml b/backend/components/twilio/helm/templates/deployments.yaml index 0e3a6a570..461dd6c30 100644 --- a/backend/components/twilio/helm/templates/deployments.yaml +++ b/backend/components/twilio/helm/templates/deployments.yaml @@ -54,6 +54,11 @@ spec: - name: Health-Check value: health-check initialDelaySeconds: 120 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.connector.resources | indent 12 }} initContainers: @@ -141,3 +146,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/viber/helm/templates/deployments.yaml b/backend/components/viber/helm/templates/deployments.yaml index ebb0d5254..55fbb10d7 100644 --- a/backend/components/viber/helm/templates/deployments.yaml +++ b/backend/components/viber/helm/templates/deployments.yaml @@ -49,6 +49,11 @@ spec: - name: Health-Check value: health-check initialDelaySeconds: 120 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.connector.resources | indent 12 }} initContainers: @@ -66,3 +71,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/webhook/helm/templates/deployments.yaml b/backend/components/webhook/helm/templates/deployments.yaml index 814a41c9a..6c6cb5b1b 100644 --- a/backend/components/webhook/helm/templates/deployments.yaml +++ b/backend/components/webhook/helm/templates/deployments.yaml @@ -55,6 +55,11 @@ spec: - name: Health-Check value: health-check initialDelaySeconds: 120 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.consumer.resources | indent 10 }} initContainers: @@ -157,3 +162,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/backend/components/whatsapp/helm/templates/deployments.yaml b/backend/components/whatsapp/helm/templates/deployments.yaml index 1fcbff4b1..4c5e04b48 100644 --- a/backend/components/whatsapp/helm/templates/deployments.yaml +++ b/backend/components/whatsapp/helm/templates/deployments.yaml @@ -113,6 +113,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.eventsRouter.resources | indent 10 }} initContainers: @@ -130,3 +135,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/docs/docs/getting-started/installation/helm.md b/docs/docs/getting-started/installation/helm.md index e47ed9b44..4d3738cc0 100644 --- a/docs/docs/getting-started/installation/helm.md +++ b/docs/docs/getting-started/installation/helm.md @@ -290,6 +290,39 @@ Run the following command to create the `Airy` platform without the bundled inst helm install airy airy/airy --timeout 10m --set prerequisites.kafka.enabled=false --values ./airy.yaml ``` +#### Confluent + +To connect to a Kafka instance in Confluent cloud, settings the `config.kafka.brokers` and `config.kafka.aurhJaas` is enough, prior to deploying the Helm chart. + +#### Aiven + +Aiven cloud uses a keystore and truststore certificates that need to be loaded on the workloads that are connecting to Kafka. Get the necessary certificates and connection files from Aiven using the `avn` CLI and place them in a separate directory. + +``` +avn service user-kafka-java-creds {KAFKA_INSTANCE} --username {USERNAME} -d ./aiven/ --password {PASSWORD} +``` + +Create a Kubernetes ConfigMap that contains the contents of the created directory: + +``` +kubectl create configmap kafka-config-certs --from-file aiven/ +``` + +Set the connection appropriate parameters in your `airy.yaml` file: + +```yaml +config: + kafka: + brokers: "the-aiven-kafka-broker-url" + keyTrustSecret: "the-key-trust-secret" +``` + +Then install Airy with the following command: + +```sh +helm install airy airy/airy --timeout 10m --set prerequisites.kafka.enabled=false --set global.kafkaCertAuth=true --values ./airy.yaml +``` + ### Kafka partitions per topic Currently all the default topics in the Airy instance are created with 10 partitions. To create these topics with a different number of partitions, add the following to your `airy.yaml` file before running `helm install` (before the initial creation of the topics): diff --git a/infrastructure/helm-chart/templates/components/api-admin/deployment.yaml b/infrastructure/helm-chart/templates/components/api-admin/deployment.yaml index 75cab9b8d..3af2e1344 100644 --- a/infrastructure/helm-chart/templates/components/api-admin/deployment.yaml +++ b/infrastructure/helm-chart/templates/components/api-admin/deployment.yaml @@ -60,6 +60,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.components.api.admin.resources | indent 10 }} initContainers: @@ -77,3 +82,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/infrastructure/helm-chart/templates/components/api-communication/deployment.yaml b/infrastructure/helm-chart/templates/components/api-communication/deployment.yaml index 362428ee8..a461864d1 100644 --- a/infrastructure/helm-chart/templates/components/api-communication/deployment.yaml +++ b/infrastructure/helm-chart/templates/components/api-communication/deployment.yaml @@ -45,6 +45,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.components.api.communication.resources | indent 10 }} initContainers: @@ -62,3 +67,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/infrastructure/helm-chart/templates/components/api-components-installer/deployment.yaml b/infrastructure/helm-chart/templates/components/api-components-installer/deployment.yaml index fd865b890..363863faf 100644 --- a/infrastructure/helm-chart/templates/components/api-components-installer/deployment.yaml +++ b/infrastructure/helm-chart/templates/components/api-components-installer/deployment.yaml @@ -81,6 +81,11 @@ spec: initialDelaySeconds: 60 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.components.api.components.installer.resources | indent 10 }} initContainers: @@ -102,4 +107,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts - +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/infrastructure/helm-chart/templates/components/api-websocket/deployment.yaml b/infrastructure/helm-chart/templates/components/api-websocket/deployment.yaml index 6739cfe3c..c4d95e0c5 100644 --- a/infrastructure/helm-chart/templates/components/api-websocket/deployment.yaml +++ b/infrastructure/helm-chart/templates/components/api-websocket/deployment.yaml @@ -45,6 +45,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.components.api.websocket.resources | indent 10 }} initContainers: @@ -62,3 +67,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/infrastructure/helm-chart/templates/components/unread-counter/deployment.yaml b/infrastructure/helm-chart/templates/components/unread-counter/deployment.yaml index c8650099f..4b4421256 100644 --- a/infrastructure/helm-chart/templates/components/unread-counter/deployment.yaml +++ b/infrastructure/helm-chart/templates/components/unread-counter/deployment.yaml @@ -45,6 +45,11 @@ spec: initialDelaySeconds: 120 periodSeconds: 10 failureThreshold: 3 +{{ if .Values.global.kafkaCertAuth }} + volumeMounts: + - name: kafka-config-certs + mountPath: /opt/kafka/certs +{{ end }} resources: {{ toYaml .Values.components.api.unread_counter.resources | indent 10 }} initContainers: @@ -62,3 +67,8 @@ spec: - name: provisioning-scripts configMap: name: provisioning-scripts +{{ if .Values.global.kafkaCertAuth }} + - name: kafka-config-certs + configMap: + name: kafka-config-certs +{{ end }} \ No newline at end of file diff --git a/infrastructure/helm-chart/templates/config/kafka.yaml b/infrastructure/helm-chart/templates/config/kafka.yaml index a00933d80..23897e721 100644 --- a/infrastructure/helm-chart/templates/config/kafka.yaml +++ b/infrastructure/helm-chart/templates/config/kafka.yaml @@ -13,3 +13,4 @@ data: {{- end }} KAFKA_SCHEMA_REGISTRY_URL: {{ .Values.config.kafka.schemaRegistryUrl }} KAFKA_COMMIT_INTERVAL_MS: "{{ .Values.config.kafka.commitInterval }}" + KAFKA_KEY_TRUST_SECRET: {{ .Values.config.kafka.keyTrustSecret }} diff --git a/infrastructure/helm-chart/values.yaml b/infrastructure/helm-chart/values.yaml index e4f6afa18..982a736b5 100644 --- a/infrastructure/helm-chart/values.yaml +++ b/infrastructure/helm-chart/values.yaml @@ -13,6 +13,7 @@ config: brokers: "kafka-headless:9092" zookeepers: "zookeeper:2181" authJaas: "" + keyTrustSecret: "" minimumReplicas: 1 schemaRegistryUrl: "http://schema-registry:8081" commitInterval: 1000 diff --git a/lib/java/kafka/core/src/main/java/co/airy/kafka/core/KafkaConsumerWrapper.java b/lib/java/kafka/core/src/main/java/co/airy/kafka/core/KafkaConsumerWrapper.java index b5739d950..ca75dc902 100644 --- a/lib/java/kafka/core/src/main/java/co/airy/kafka/core/KafkaConsumerWrapper.java +++ b/lib/java/kafka/core/src/main/java/co/airy/kafka/core/KafkaConsumerWrapper.java @@ -12,6 +12,10 @@ import java.time.temporal.ChronoUnit; import java.util.Collection; import java.util.Properties; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.config.SslConfigs; +import java.util.HashMap; + public class KafkaConsumerWrapper { @@ -22,6 +26,7 @@ public class KafkaConsumerWrapper { private KafkaConsumer consumer; private String jaasConfig; + private String kafkaKeyTrustSecret; public KafkaConsumerWrapper(final String brokers, final String schemaRegistryUrl) { props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, brokers); @@ -33,13 +38,22 @@ public KafkaConsumerWrapper(final String brokers, final String schemaRegistryUrl props.put(KafkaAvroDeserializerConfig.SPECIFIC_AVRO_READER_CONFIG, true); } - public KafkaConsumerWrapper withAuthJaas(String jaasConfig) { + public KafkaConsumerWrapper withAuthJaas(String jaasConfig, String kafkaKeyTrustSecret) { this.jaasConfig = jaasConfig; if(jaasConfig != null) { props.put("security.protocol", "SASL_SSL"); props.put("sasl.mechanism", "PLAIN"); props.put("sasl.jaas.config", jaasConfig); } + if (kafkaKeyTrustSecret != null) { + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL"); + props.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, "/opt/kafka/certs/client.truststore.jks"); + props.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, kafkaKeyTrustSecret); + props.put(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, "PKCS12"); + props.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, "/opt/kafka/certs/client.keystore.p12"); + props.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, kafkaKeyTrustSecret); + props.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, kafkaKeyTrustSecret); + } return this; } diff --git a/lib/java/kafka/streams/src/main/java/co/airy/kafka/streams/KafkaStreamsWrapper.java b/lib/java/kafka/streams/src/main/java/co/airy/kafka/streams/KafkaStreamsWrapper.java index f44f5075a..18ca499ca 100644 --- a/lib/java/kafka/streams/src/main/java/co/airy/kafka/streams/KafkaStreamsWrapper.java +++ b/lib/java/kafka/streams/src/main/java/co/airy/kafka/streams/KafkaStreamsWrapper.java @@ -30,6 +30,10 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.config.SslConfigs; +import java.util.HashMap; + public class KafkaStreamsWrapper { private static final Logger log = AiryLoggerFactory.getLogger(KafkaStreamsWrapper.class); @@ -37,6 +41,7 @@ public class KafkaStreamsWrapper { private final String brokers; private final String schemaRegistryUrl; private String jaasConfig; + private String kafkaKeyTrustSecret; private long commitIntervalInMs; private long suppressIntervalInMs; private int threadCount; @@ -70,8 +75,9 @@ public KafkaStreamsWrapper(final String brokers, final String schemaRegistryUrl) healthCheckRunnerThread = new HealthCheckRunner(testMode); } - public KafkaStreamsWrapper withJaasConfig(String jaasConfig) { + public KafkaStreamsWrapper withJaasConfig(String jaasConfig, String kafkaKeyTrustSecret) { this.jaasConfig = jaasConfig; + this.kafkaKeyTrustSecret = kafkaKeyTrustSecret; return this; } @@ -227,6 +233,17 @@ public synchronized void start(final Topology topology, final String appId) thro props.put("sasl.jaas.config", jaasConfig); } + if (this.kafkaKeyTrustSecret != null) { + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL"); + props.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, "/opt/kafka/certs/client.truststore.jks"); + props.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, kafkaKeyTrustSecret); + props.put(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, "PKCS12"); + props.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, "/opt/kafka/certs/client.keystore.p12"); + props.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, kafkaKeyTrustSecret); + props.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, kafkaKeyTrustSecret); + } + + props.put(ProducerConfig.MAX_REQUEST_SIZE_CONFIG, this.maxRequestSize); props.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, this.fetchMaxBytes); diff --git a/lib/java/spring/kafka/core/src/main/java/co/airy/spring/kafka/core/KafkaCoreConfig.java b/lib/java/spring/kafka/core/src/main/java/co/airy/spring/kafka/core/KafkaCoreConfig.java index 0cd6c2812..f8488e2f6 100644 --- a/lib/java/spring/kafka/core/src/main/java/co/airy/spring/kafka/core/KafkaCoreConfig.java +++ b/lib/java/spring/kafka/core/src/main/java/co/airy/spring/kafka/core/KafkaCoreConfig.java @@ -13,6 +13,9 @@ import org.springframework.context.annotation.Scope; import java.util.Properties; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.config.SslConfigs; +import java.util.HashMap; @Configuration @PropertySource("classpath:kafka-core.properties") @@ -21,7 +24,8 @@ public class KafkaCoreConfig { @Lazy @Scope("prototype") public KafkaProducer kafkaProducer(@Value("${kafka.brokers}") final String brokers, @Value("${kafka.schema-registry-url}") final String schemaRegistryUrl, - @Value("${AUTH_JAAS:#{null}}") final String jaasConfig) { + @Value("${AUTH_JAAS:#{null}}") final String jaasConfig, + @Value("${KAFKA_KEY_TRUST_SECRET:#{null}}") final String kafkaKeyTrustSecret) { final Properties props = new Properties(); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokers); @@ -37,6 +41,16 @@ public KafkaProducer kafkaProducer(@Value("${kafka.brokers}") final props.put("sasl.jaas.config", jaasConfig); } + if (kafkaKeyTrustSecret != null) { + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL"); + props.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, "/opt/kafka/certs/client.truststore.jks"); + props.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, kafkaKeyTrustSecret); + props.put(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, "PKCS12"); + props.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, "/opt/kafka/certs/client.keystore.p12"); + props.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, kafkaKeyTrustSecret); + props.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, kafkaKeyTrustSecret); + } + return new KafkaProducer<>(props); } @@ -44,8 +58,9 @@ public KafkaProducer kafkaProducer(@Value("${kafka.brokers}") final @Lazy @Scope("prototype") public KafkaConsumerWrapper kafkaConsumer(@Value("${kafka.brokers}") final String brokers, @Value("${kafka.schema-registry-url}") final String schemaRegistryUrl, - @Value("${kafka.sasl.jaas.config:#{null}}") final String jaasConfig) { + @Value("${kafka.sasl.jaas.config:#{null}}") final String jaasConfig, + @Value("${KAFKA_KEY_TRUST_SECRET:#{null}}") final String kafkaKeyTrustSecret) { return new KafkaConsumerWrapper(brokers, schemaRegistryUrl) - .withAuthJaas(jaasConfig); + .withAuthJaas(jaasConfig, kafkaKeyTrustSecret); } } diff --git a/lib/java/spring/kafka/streams/src/main/java/co/airy/spring/kafka/streams/KafkaStreamsConfig.java b/lib/java/spring/kafka/streams/src/main/java/co/airy/spring/kafka/streams/KafkaStreamsConfig.java index f9995abfb..217f2f9a6 100644 --- a/lib/java/spring/kafka/streams/src/main/java/co/airy/spring/kafka/streams/KafkaStreamsConfig.java +++ b/lib/java/spring/kafka/streams/src/main/java/co/airy/spring/kafka/streams/KafkaStreamsConfig.java @@ -30,6 +30,9 @@ public class KafkaStreamsConfig { @Value("${AUTH_JAAS:#{null}}") private String jaasConfig; + @Value("${KAFKA_KEY_TRUST_SECRET:#{null}}") + private String kafkaKeyTrustSecret; + @Value("${kafka.rpc-port:0}") private int rpcPort; @@ -68,7 +71,7 @@ public class KafkaStreamsConfig { public KafkaStreamsWrapper airyKafkaStreams(@Value("${kafka.brokers}") final String brokers, @Value("${kafka.schema-registry-url}") final String schemaRegistryUrl) { return new KafkaStreamsWrapper(brokers, schemaRegistryUrl) .withCommitIntervalInMs(commitIntervalMs) - .withJaasConfig(jaasConfig) + .withJaasConfig(jaasConfig, kafkaKeyTrustSecret) .withSuppressIntervalInMs(suppressIntervalMs) .withThreadCount(streamsThreadCount) .withAppServerHost(rpcHost) From 397e04be7d63fe9091b44974158344b6201e6c21 Mon Sep 17 00:00:00 2001 From: ljupcovangelski Date: Thu, 9 May 2024 11:49:53 +0200 Subject: [PATCH 4/4] Update changelog # --- docs/docs/changelog.md | 93 ++++++++++++++++++++++++++---------------- 1 file changed, 58 insertions(+), 35 deletions(-) diff --git a/docs/docs/changelog.md b/docs/docs/changelog.md index 0b0cb793a..9d4aa94e5 100644 --- a/docs/docs/changelog.md +++ b/docs/docs/changelog.md @@ -3,6 +3,64 @@ title: Changelog sidebar_label: 📝 Changelog --- +## 0.55.0 + +#### Changes + +- [[#4167](https://github.com/airyhq/airy/issues/4167)] Kafka certificate authentication [[#4168](https://github.com/airyhq/airy/pull/4168)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Change name for Pinecone connector [[#4127](https://github.com/airyhq/airy/pull/4127)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Add description for components [[#4126](https://github.com/airyhq/airy/pull/4126)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Add LLM filter in Catalog [[#4125](https://github.com/airyhq/airy/pull/4125)] + +#### 🚀 Features + +- [[#4139](https://github.com/airyhq/airy/issues/4139)] Add Flink connector [[#4147](https://github.com/airyhq/airy/pull/4147)] +- [[#3945](https://github.com/airyhq/airy/issues/3945)] Slack connector [[#4146](https://github.com/airyhq/airy/pull/4146)] +- [[#4144](https://github.com/airyhq/airy/issues/4144)] Adapt Frontend with Schema Manager [[#4145](https://github.com/airyhq/airy/pull/4145)] +- [[#4141](https://github.com/airyhq/airy/issues/4141)] Schema Registry Manager [[#4143](https://github.com/airyhq/airy/pull/4143)] +- [[#4137](https://github.com/airyhq/airy/issues/4137)] Improve Kafka Sections [[#4138](https://github.com/airyhq/airy/pull/4138)] +- [[#4132](https://github.com/airyhq/airy/issues/4132)] Added Copilot source to libs and apps [[#4133](https://github.com/airyhq/airy/pull/4133)] + +#### 🐛 Bug Fixes + +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Fix selecting fields in streams [[#4123](https://github.com/airyhq/airy/pull/4123)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Fix symbol [[#4122](https://github.com/airyhq/airy/pull/4122)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Update icons for LLM components [[#4120](https://github.com/airyhq/airy/pull/4120)] +- [[#4108](https://github.com/airyhq/airy/issues/4108)] Fix schemas and topics screens [[#4109](https://github.com/airyhq/airy/pull/4109)] + +#### 📚 Documentation + +- [[#4134](https://github.com/airyhq/airy/issues/4134)] Update the diagram in the README file [[#4136](https://github.com/airyhq/airy/pull/4136)] +- [[#4134](https://github.com/airyhq/airy/issues/4134)] Update main diagram [[#4135](https://github.com/airyhq/airy/pull/4135)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Improve docs [[#4131](https://github.com/airyhq/airy/pull/4131)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Improve docs [[#4128](https://github.com/airyhq/airy/pull/4128)] +- [[#4156](https://github.com/airyhq/airy/issues/4156)] Docs - Remove Solutions link [[#4115](https://github.com/airyhq/airy/pull/4115)] + +#### 🧰 Maintenance + +- Bump google.golang.org/protobuf from 1.28.0 to 1.33.0 [[#4155](https://github.com/airyhq/airy/pull/4155)] +- Bump webpack-dev-middleware from 5.3.3 to 5.3.4 in /docs [[#4154](https://github.com/airyhq/airy/pull/4154)] +- Bump google.golang.org/protobuf from 1.28.0 to 1.33.0 in /infrastructure/lib/go/k8s/handler [[#4152](https://github.com/airyhq/airy/pull/4152)] +- Bump webpack-dev-middleware from 5.3.3 to 5.3.4 [[#4153](https://github.com/airyhq/airy/pull/4153)] +- Bump express from 4.18.2 to 4.19.2 [[#4151](https://github.com/airyhq/airy/pull/4151)] +- Bump express from 4.18.2 to 4.19.2 in /docs [[#4150](https://github.com/airyhq/airy/pull/4150)] +- Bump follow-redirects from 1.15.2 to 1.15.6 [[#4149](https://github.com/airyhq/airy/pull/4149)] +- Bump follow-redirects from 1.15.2 to 1.15.6 in /docs [[#4148](https://github.com/airyhq/airy/pull/4148)] +- Bump github.com/cyphar/filepath-securejoin from 0.2.3 to 0.2.4 [[#4118](https://github.com/airyhq/airy/pull/4118)] +- Bump @adobe/css-tools from 4.0.1 to 4.3.1 [[#4116](https://github.com/airyhq/airy/pull/4116)] +- Bump @svgr/plugin-svgo from 6.5.1 to 8.1.0 [[#4114](https://github.com/airyhq/airy/pull/4114)] +- Bump semver from 5.7.1 to 5.7.2 in /docs [[#4110](https://github.com/airyhq/airy/pull/4110)] +- Bump sass-loader from 13.1.0 to 13.3.2 [[#4103](https://github.com/airyhq/airy/pull/4103)] +- Bump word-wrap from 1.2.3 to 1.2.4 [[#4112](https://github.com/airyhq/airy/pull/4112)] + +#### Airy CLI + +You can download the Airy CLI for your operating system from the following links: + +[MacOS](https://airy-core-binaries.s3.amazonaws.com/0.55.0/darwin/amd64/airy) +[Linux](https://airy-core-binaries.s3.amazonaws.com/0.55.0/linux/amd64/airy) +[Windows](https://airy-core-binaries.s3.amazonaws.com/0.55.0/windows/amd64/airy.exe) + ## 0.54.0 #### 🚀 Features @@ -1348,38 +1406,3 @@ You can download the Airy CLI for your operating system from the following links [Linux](https://airy-core-binaries.s3.amazonaws.com/0.35.0/linux/amd64/airy) [Windows](https://airy-core-binaries.s3.amazonaws.com/0.35.0/windows/amd64/airy.exe) -## 0.34.0 - -#### Changes - -#### 🚀 Features - -- [[#2518](https://github.com/airyhq/airy/issues/2518)] Add fargate annotation [[#2540](https://github.com/airyhq/airy/pull/2540)] -- [[#2305](https://github.com/airyhq/airy/issues/2305)] Add CLI outdated version warning [[#2529](https://github.com/airyhq/airy/pull/2529)] - -#### 🐛 Bug Fixes - -- [[#2434](https://github.com/airyhq/airy/issues/2434)] Fix broken instagram Facebook inbox ingestion [[#2535](https://github.com/airyhq/airy/pull/2535)] -- [[#2457](https://github.com/airyhq/airy/issues/2457)] Fix upgrade to same version [[#2538](https://github.com/airyhq/airy/pull/2538)] -- [[#2510](https://github.com/airyhq/airy/issues/2510)] Improve error logging for helm install [[#2522](https://github.com/airyhq/airy/pull/2522)] -- [[#2255](https://github.com/airyhq/airy/issues/2255)] Fix helm chart url [[#2525](https://github.com/airyhq/airy/pull/2525)] -- [[#2523](https://github.com/airyhq/airy/issues/2523)] Fix VERSION and add changelog [[#2524](https://github.com/airyhq/airy/pull/2524)] -- [[#2473](https://github.com/airyhq/airy/issues/2473)] fix failing cypress test [[#2507](https://github.com/airyhq/airy/pull/2507)] - -#### 🧰 Maintenance - -- Bump react-redux from 7.2.5 to 7.2.6 [[#2539](https://github.com/airyhq/airy/pull/2539)] -- Bump reselect from 4.0.0 to 4.1.1 [[#2533](https://github.com/airyhq/airy/pull/2533)] -- Bump sass-loader from 12.1.0 to 12.3.0 [[#2534](https://github.com/airyhq/airy/pull/2534)] -- Bump @types/react-dom from 17.0.9 to 17.0.10 [[#2526](https://github.com/airyhq/airy/pull/2526)] -- Bump react-markdown from 7.0.1 to 7.1.0 [[#2527](https://github.com/airyhq/airy/pull/2527)] -- Bump webpack from 5.54.0 to 5.59.1 [[#2517](https://github.com/airyhq/airy/pull/2517)] - -#### Airy CLI - -You can download the Airy CLI for your operating system from the following links: - -[MacOS](https://airy-core-binaries.s3.amazonaws.com/0.34.0/darwin/amd64/airy) -[Linux](https://airy-core-binaries.s3.amazonaws.com/0.34.0/linux/amd64/airy) -[Windows](https://airy-core-binaries.s3.amazonaws.com/0.34.0/windows/amd64/airy.exe) -