Merge branch 'master' into fix/snappy-dep

.github/workflows/build.yml

@@ -7,8 +7,8 @@ on:
     branches:
       - master
 env:
-  OTP_VERSION: "24.1"
-  REBAR_VERSION: "3.17.0"
+  OTP_VERSION: "26"
+  REBAR_VERSION: "3.20.0"
 
 jobs:
   lint:
@@ -44,8 +44,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        otp: ["24.1", "23.3.4.7", "22.3.4.21"]
-        kafka: ["2.4", "1.1", "0.11"]
+        otp: ["26"]
+        kafka: ["0.9", "0.10", "0.11", "2.8", "1.1", "3.6"]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -69,7 +69,8 @@ jobs:
         run: |
           export KAFKA_VERSION=${{ matrix.kafka }}
           echo "Running Kafka ${KAFKA_VERSION}"
-          scripts/setup-test-env.sh && rebar3 do ct,eunit
+          make test-env
+          make t
       - name: Store test logs
         uses: actions/upload-artifact@v1
         if: always()

.gitignore

@@ -23,4 +23,6 @@ _rel/
 *.log
 relx
 docker/
-TAGS
+TAGS
+.vscode/
+test/data/ssl/*.pem

CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+- 3.19.1
+  - Made brod-cli to work on OTP 26. [PR#582](https://github.com/kafka4beam/brod/pull/582)
+    - `--ssl` option is now mandatory if TLS is to be used (previously it can be derived from `--cacertfile` option)
+    - TLS version defaults to 1.2, added `--ssl-versions` to support explictly setting TLS 1.3
+
+- 3.19.0
+  - Forward unhandled messages in topic/group consumer processes to handle_info/2 callbacks
+    in order to support arbitrary message passing [PR#580](https://github.com/kafka4beam/brod/pull/580)
+
+- 3.18.0
+  - Add transactional APIs. [PR#549](https://github.com/kafka4beam/brod/pull/549)
+  - Fix unnecessary group coordinator restart due to `hb_timeout` exception. [PR#578](https://github.com/kafka4beam/brod/pull/578)
+  - Changed supervisor3 progress log level from `info` to `debug`. [PR#572](https://github.com/kafka4beam/brod/pull/572)
+  - Type spec fix. [PR#571](https://github.com/kafka4beam/brod/pull/571)
+  - Remove unused macro. [PR#575](https://github.com/kafka4beam/brod/pull/575)
+
 - 3.17.1
   - Upgrade `kafka_protocol` from 4.1.3 to 4.1.5
     - Allow space after `,` in comma-separated bootstrapping host:port list

Makefile

@@ -1,3 +1,5 @@
+KAFKA_VERSION ?= 3.6
+export KAFKA_VERSION
 all: compile
 
 compile:
@@ -8,6 +10,10 @@ lint:
 
 test-env:
 	@./scripts/setup-test-env.sh
+	@mkdir -p ./test/data/ssl
+	@docker cp kafka-1:/localhost-ca-crt.pem ./test/data/ssl/ca.pem
+	@docker cp kafka-1:/localhost-client-key.pem ./test/data/ssl/client-key.pem
+	@docker cp kafka-1:/localhost-client-crt.pem ./test/data/ssl/client-crt.pem
 
 ut:
 	@rebar3 eunit -v --cover_export_name ut-$(KAFKA_VERSION)

guides/examples/Authentication.md

@@ -60,6 +60,7 @@ For more info see the Erlang Ecosystem Foundation's [server certificate verifica
                      , { depth, 3 }
                      , { customize_hostname_check,
                           [{match_fun, public_key:pkix_verify_hostname_match_fun(https)}]}
+                     , {version, ['tlsv1.3', 'tlsv1.2']}
                      ]}
             , { sasl, {plain, "GFRW5BSQHKEH0TSG", "GrL3CNTkLhsvtBr8srGn0VilMpgDb4lPD"}}
             ]

rebar.config

@@ -16,9 +16,9 @@
     {relx, [{release, {brod, "i"}, % release the interactive shell as brod-i
       [brod, jsone, docopt]},
       {include_erts, true},
-      {overlay, [{copy, "scripts/brod", "bin"},
-        {copy, "{{lib_dirs}}/crc32cer/priv/crc32cer*.so", "bin"},
-        {copy, "{{lib_dirs}}/snappyer/priv/snappyer.so", "bin"}
+      {overlay, [{copy, "scripts/brod", "bin/"},
+        {copy, "{{lib_dirs}}/crc32cer/priv/crc32cer*.so", "bin/"},
+        {copy, "{{lib_dirs}}/snappyer/priv/snappyer.so", "bin/"}
       ]}
     ]}]},
   {test, [
@@ -27,8 +27,8 @@
            , {jsone, "1.7.0"}
            , {meck, "0.9.2"}
            , {proper, "1.4.0"}
-           , {snabbkaffe, "1.0.1"}
            , {snappyer, "1.2.8"}
+           , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {branch, "1.0.8"}}}
     ]},
     {erl_opts, [warnings_as_errors, {d, build_brod_cli}]}
   ]}

scripts/docker-compose.yml

@@ -1,35 +1,17 @@
 version: "2"
 
 services:
-  pause:
-    image: "gcr.io/google_containers/pause-amd64:3.0"
-    networks:
-      - pausenet
-    ports:
-      - "2181:2181"
-      - "9092:9092"
-      - "9093:9093"
-      - "9094:9094"
-      - "9095:9095"
-      - "9192:9192"
-      - "9193:9193"
-      - "9194:9194"
-      - "9195:9195"
-    container_name: pause
   zookeeper:
-    depends_on:
-      - pause
-    image: "zmstone/kafka:${KAFKA_VERSION}"
+    image: "zmstone/kafka:${KAFKA_IMAGE_VERSION}"
     container_name: zookeeper
     command: run zookeeper
-    network_mode: service:pause
+    network_mode: host
   kafka_1:
     depends_on:
-      - pause
       - zookeeper
-    image: "zmstone/kafka:${KAFKA_VERSION}"
+    image: "zmstone/kafka:${KAFKA_IMAGE_VERSION}"
     container_name: "kafka-1"
-    network_mode: service:pause
+    network_mode: host
     environment:
       BROKER_ID: 0
       PLAINTEXT_PORT: 9092
@@ -40,11 +22,10 @@ services:
       ZOOKEEPER_CONNECT: "localhost:2181"
   kafka_2:
     depends_on:
-      - pause
       - zookeeper
-    image: "zmstone/kafka:${KAFKA_VERSION}"
+    image: "zmstone/kafka:${KAFKA_IMAGE_VERSION}"
     container_name: "kafka-2"
-    network_mode: service:pause
+    network_mode: host
     environment:
       BROKER_ID: 1
       PLAINTEXT_PORT: 9192
@@ -53,6 +34,3 @@ services:
       SASL_PLAINTEXT_PORT: 9195
       ADVERTISED_HOSTNAME: localhost
       ZOOKEEPER_CONNECT: "localhost:2181"
-
-networks:
-  pausenet:

scripts/setup-test-env.sh

@@ -1,5 +1,9 @@
 #!/bin/bash -eu
 
+if [ -n "${DEBUG:-}" ]; then
+    set -x
+fi
+
 docker ps > /dev/null || {
     echo "You must be a member of docker group to run this script"
     exit 1
@@ -18,46 +22,77 @@ function docker_compose {
     fi
 }
 
-VERSION=${KAFKA_VERSION:-1.1}
-if [ -z $VERSION ]; then VERSION=$1; fi
+KAFKA_VERSION=${KAFKA_VERSION:-3.6}
+if [ -z $KAFKA_VERSION ]; then KAFKA_VERSION=$1; fi
 
-case $VERSION in
+case $KAFKA_VERSION in
+  0.9*)
+    KAFKA_VERSION="0.9";;
   0.10*)
-    VERSION="0.10";;
+    KAFKA_VERSION="0.10";;
   0.11*)
-    VERSION="0.11";;
+    KAFKA_VERSION="0.11";;
   1.*)
-    VERSION="1.1";;
+    KAFKA_VERSION="1.1";;
   2.*)
-    VERSION="2.4";;
+    KAFKA_VERSION="2.8";;
+  3.*)
+    KAFKA_VERSION="3.6";;
   *)
-    VERSION="2.4";;
+    KAFKA_VERSION="3.6";;
 esac
 
-echo "Using KAFKA_VERSION=$VERSION"
-export KAFKA_VERSION=$VERSION
+export KAFKA_IMAGE_VERSION="1.1-${KAFKA_VERSION}"
+echo "env KAFKA_IMAGE_VERSION=$KAFKA_IMAGE_VERSION"
 
 TD="$(cd "$(dirname "$0")" && pwd)"
 
 docker_compose -f $TD/docker-compose.yml down || true
 docker_compose -f $TD/docker-compose.yml up -d
 
+if [[ "$KAFKA_VERSION" == 2* ]] || [[ "$KAFKA_VERSION" == 3* ]]; then
+  MAYBE_ZOOKEEPER="--bootstrap-server localhost:9092"
+else
+  MAYBE_ZOOKEEPER="--zookeeper localhost:2181"
+fi
 
-n=0
-while [ "$(docker exec kafka-1 bash -c '/opt/kafka/bin/kafka-topics.sh --zookeeper localhost --list')" != '' ]; do
-  if [ $n -gt 4 ]; then
-    echo "timeout waiting for kakfa_1"
-    exit 1
+TOPIC_LIST_CMD="/opt/kafka/bin/kafka-topics.sh $MAYBE_ZOOKEEPER --list"
+MAX_WAIT_SEC=10
+
+function wait_for_kafka {
+  local which_kafka="$1"
+  local n=0
+  local port=':9092'
+  local topic_list listener
+  if [ "$which_kafka" = 'kafka-2' ]; then
+    port=':9192'
   fi
-  n=$(( n + 1 ))
-  sleep 1
-done
+  while true; do
+    listener="$(netstat -tnlp 2>&1 | grep $port || true)"
+    if [ "$listener" != '' ]; then
+      topic_list="$(docker exec $which_kafka $TOPIC_LIST_CMD 2>&1)"
+      if [ "${topic_list-}" = '' ]; then
+          break
+      fi
+    fi
+    if [ $n -gt $MAX_WAIT_SEC ]; then
+      echo "timeout waiting for kafka-1"
+      echo "last print: ${topic_list:-}"
+      exit 1
+    fi
+    n=$(( n + 1 ))
+    sleep 1
+  done
+}
+
+wait_for_kafka kafka-1
+wait_for_kafka kafka-2
 
 function create_topic {
   TOPIC_NAME="$1"
   PARTITIONS="${2:-1}"
   REPLICAS="${3:-1}"
-  CMD="/opt/kafka/bin/kafka-topics.sh --zookeeper localhost --create --partitions $PARTITIONS --replication-factor $REPLICAS --topic $TOPIC_NAME --config min.insync.replicas=1"
+  CMD="/opt/kafka/bin/kafka-topics.sh $MAYBE_ZOOKEEPER --create --partitions $PARTITIONS --replication-factor $REPLICAS --topic $TOPIC_NAME --config min.insync.replicas=1"
   docker exec kafka-1 bash -c "$CMD"
 }
 
@@ -80,7 +115,7 @@ create_topic "brod_compression_SUITE"
 create_topic "lz4-test"
 create_topic "test-topic"
 
-if [[ "$KAFKA_VERSION" = 2* ]]; then
+if [[ "$KAFKA_VERSION" = 2* ]] || [[ "$KAFKA_VERSION" = 3* ]]; then
   MAYBE_NEW_CONSUMER=""
 else
   MAYBE_NEW_CONSUMER="--new-consumer"
@@ -90,5 +125,5 @@ docker exec kafka-1 /opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server l
 
 # for kafka 0.11 or later, add sasl-scram test credentials
 if [[ "$KAFKA_VERSION" != 0.9* ]] && [[ "$KAFKA_VERSION" != 0.10* ]]; then
-  docker exec kafka-1 /opt/kafka/bin/kafka-configs.sh --zookeeper localhost:2181 --alter --add-config 'SCRAM-SHA-256=[iterations=8192,password=ecila],SCRAM-SHA-512=[password=ecila]' --entity-type users --entity-name alice
+  docker exec kafka-1 /opt/kafka/bin/kafka-configs.sh $MAYBE_ZOOKEEPER --alter --add-config 'SCRAM-SHA-256=[iterations=8192,password=ecila],SCRAM-SHA-512=[password=ecila]' --entity-type users --entity-name alice
 fi