just testing

metricbeat/docker-compose.yml

@@ -1,5 +1,3 @@
-version: '2.3'
-
 services:
   beat:
     build: ${PWD}/.

metricbeat/module/kafka/_meta/Dockerfile

@@ -18,7 +18,7 @@ RUN apt-get update && apt-get install -y curl openjdk-8-jre-headless netcat-open
 RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && \
     curl -J -L -s -f -o - https://github.com/kadwanev/retry/releases/download/1.0.1/retry-1.0.1.tar.gz | tar xfz - -C /usr/local/bin && \
     retry --min 1 --max 180 -- curl -J -L -s -f --show-error -o $INSTALL_DIR/kafka.tgz \
-        "https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz" && \
+        "https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_2.12-${KAFKA_VERSION}.tgz" && \
     tar xzf ${INSTALL_DIR}/kafka.tgz -C ${KAFKA_HOME} --strip-components 1
 
 RUN retry --min 1 --max 180 -- curl -J -L -s -f --show-error -o /opt/jolokia-jvm-1.5.0-agent.jar \

metricbeat/module/kafka/_meta/healthcheck.sh

@@ -4,11 +4,11 @@
 
 TOPIC="foo-`date '+%s-%N'`"
 
-${KAFKA_HOME}/bin/kafka-topics.sh --zookeeper=127.0.0.1:2181 --create --partitions 1 --topic "${TOPIC}" --replication-factor 1
+${KAFKA_HOME}/bin/kafka-topics.sh --bootstrap-server localhost:9091 --create --partitions 1 --topic "${TOPIC}" --replication-factor 1
 rc=$?
 if [[ $rc != 0 ]]; then
 	exit $rc
 fi
 
-${KAFKA_HOME}/bin/kafka-topics.sh --zookeeper=127.0.0.1:2181 --delete --topic "${TOPIC}"
+${KAFKA_HOME}/bin/kafka-topics.sh --bootstrap-server localhost:9091 --delete --topic "${TOPIC}"
 exit 0

metricbeat/module/kafka/_meta/run.sh

@@ -34,7 +34,7 @@ wait_for_port() {
     nc -z localhost $port
 }
 
-${KAFKA_HOME}/bin/kafka-topics.sh --zookeeper=127.0.0.1:2181 --create --partitions 1 --topic test --replication-factor 1
+${KAFKA_HOME}/bin/kafka-topics.sh  --bootstrap-server localhost:9091 --create --partitions 1 --topic test --replication-factor 1
 
 echo "Starting ZooKeeper"
 ${KAFKA_HOME}/bin/zookeeper-server-start.sh ${KAFKA_HOME}/config/zookeeper.properties &
@@ -44,7 +44,7 @@ echo "Starting Kafka broker"
 mkdir -p ${KAFKA_LOGS_DIR}
 export KAFKA_OPTS="-Djava.security.auth.login.config=/etc/kafka/server_jaas.conf -javaagent:/opt/jolokia-jvm-1.5.0-agent.jar=port=8779,host=0.0.0.0"
 ${KAFKA_HOME}/bin/kafka-server-start.sh ${KAFKA_HOME}/config/server.properties \
-    --override authorizer.class.name=kafka.security.auth.SimpleAclAuthorizer \
+    --override authorizer.class.name=kafka.security.authorizer.AclAuthorizer \
     --override super.users=User:admin \
     --override sasl.enabled.mechanisms=PLAIN \
     --override sasl.mechanism.inter.broker.protocol=PLAIN \
@@ -62,12 +62,12 @@ wait_for_port 8779
 echo "Kafka load status code $?"
 
 # ACLS used to prepare tests
-${KAFKA_HOME}/bin/kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --allow-principal User:producer --operation All --cluster --topic '*' --group '*'
-${KAFKA_HOME}/bin/kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --allow-principal User:consumer --operation All --cluster --topic '*' --group '*'
+${KAFKA_HOME}/bin/kafka-acls.sh --bootstrap-server localhost:9091 --add --allow-principal User:producer --operation All --cluster --topic '*' --group '*'
+${KAFKA_HOME}/bin/kafka-acls.sh --bootstrap-server localhost:9091 --add --allow-principal User:consumer --operation All --cluster --topic '*' --group '*'
 
 # Minimal ACLs required by metricbeat. If this needs to be changed, please update docs too
-${KAFKA_HOME}/bin/kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --allow-principal User:stats --operation Describe --group '*'
-${KAFKA_HOME}/bin/kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --add --allow-principal User:stats --operation Read --topic '*'
+${KAFKA_HOME}/bin/kafka-acls.sh --bootstrap-server localhost:9091 --add --allow-principal User:stats --operation Describe --group '*'
+${KAFKA_HOME}/bin/kafka-acls.sh --bootstrap-server localhost:9091 --add --allow-principal User:stats --operation Read --topic '*'
 
 touch /tmp/.acls_loaded
 

metricbeat/module/kafka/_meta/supported-versions.yml

@@ -1,4 +1,5 @@
 variants:
+  - KAFKA_VERSION: 3.6.0
   - KAFKA_VERSION: 2.2.2
   - KAFKA_VERSION: 2.1.1
   - KAFKA_VERSION: 2.0.0

metricbeat/module/kafka/consumergroup/consumergroup.go

@@ -52,7 +52,7 @@ var debugf = logp.MakeDebug("kafka")
 // New creates a new instance of the MetricSet.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	opts := kafka.MetricSetOptions{
-		Version: "2.2.0",
+		Version: "3.6.0",
 	}
 
 	ms, err := kafka.NewMetricSet(base, opts)

metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go

@@ -107,7 +107,7 @@ func startConsumer(t *testing.T, host string, groupID string) (io.Closer, error)
 	// Create a new consumer group
 	consumerGroup, err := sarama.NewConsumerGroup(brokers, groupID, config)
 	if err != nil {
-		t.Fatalf("Error creating consumer group: %v", err)
+		t.Fatalf("Error creating consumer group: %v, brokers: %s", err, brokers)
 		return nil, err
 	}
 

metricbeat/module/kafka/docker-compose.yml

@@ -1,10 +1,10 @@
 services:
   kafka:
-    image: docker.elastic.co/integrations-ci/beats-kafka:${KAFKA_VERSION:-2.2.2}-2
+    image: docker.elastic.co/integrations-ci/beats-kafka:${KAFKA_VERSION:-3.6.0}-2
     build:
       context: ./_meta
       args:
-        KAFKA_VERSION: ${KAFKA_VERSION:-2.2.2}
+        KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
     ports:
       - 9092
       - 8779

testing/environments/docker/kafka/Dockerfile

@@ -6,7 +6,7 @@ ENV KAFKA_HOME=/kafka
 ENV KAFKA_ADVERTISED_HOST=kafka
 
 ENV KAFKA_LOGS_DIR="/kafka-logs"
-ENV KAFKA_VERSION=2.2.2
+ENV KAFKA_VERSION=3.6.0
 ENV _JAVA_OPTIONS="-Djava.net.preferIPv4Stack=true"
 ENV TERM=linux
 
@@ -15,7 +15,7 @@ RUN apt-get update && apt-get install -y curl openjdk-11-jre-headless netcat-ope
 RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && \
     curl -J -L -s -f -o - https://github.com/kadwanev/retry/releases/download/1.0.1/retry-1.0.1.tar.gz | tar xfz - -C /usr/local/bin && \
     retry --min 1 --max 180 -- curl -J -L -s -f --show-error -o $INSTALL_DIR/kafka.tgz \
-        "https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz" && \
+        "https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_2.12-${KAFKA_VERSION}.tgz" && \
     tar xzf ${INSTALL_DIR}/kafka.tgz -C ${KAFKA_HOME} --strip-components 1
 
 ADD run.sh /run.sh