feature: create FIPS complaint linux image

Makefile

@@ -21,12 +21,14 @@ dev: DOCKER_BUILD_FLAGS =
 dev: release
 
 .PHONY: release
-release: build build-init linux-plugins
+release: build build-init build-fips linux-plugins linux-plugins-fips
 	docker system prune -f
 	docker build $(DOCKER_BUILD_FLAGS) -t amazon/aws-for-fluent-bit:main-release -f ./scripts/dockerfiles/Dockerfile.main-release .
 	docker tag amazon/aws-for-fluent-bit:main-release amazon/aws-for-fluent-bit:latest
 	docker system prune -f
 	docker build $(DOCKER_BUILD_FLAGS) -t amazon/aws-for-fluent-bit:init-latest -f ./scripts/dockerfiles/Dockerfile.init-release .
+	docker system prune -f
+	docker build $(DOCKER_BUILD_FLAGS) -t amazon/aws-for-fluent-bit:fips-latest -f ./scripts/dockerfiles/Dockerfile.fips-release .
 
 .PHONY: debug
 debug: main-debug init-debug
@@ -40,37 +42,39 @@ build:
 build-init:
 	docker build $(DOCKER_BUILD_FLAGS) -t amazon/aws-for-fluent-bit:build-init -f ./scripts/dockerfiles/Dockerfile.build-init .
 
+.PHONY: build-fips
+build-fips:
+	docker build $(DOCKER_BUILD_FLAGS) -t amazon/aws-for-fluent-bit:build-fips -f ./scripts/dockerfiles/Dockerfile.build-fips .
+
 #TODO: the bash script opts does not work on developer Macs
-windows-plugins: export OS_TYPE = windows
-linux-plugins: export OS_TYPE = linux
+.PHONY: build-plugins windows-plugins linux-plugins linux-plugins-fips
 
-.PHONY: windows-plugins
-windows-plugins:
+# Reusable target for building plugins
+build-plugins:
+	OS_TYPE=$(OS_TYPE) FIPS=$(FIPS) \
 	./scripts/build_plugins.sh \
-    	--KINESIS_PLUGIN_CLONE_URL=${KINESIS_PLUGIN_CLONE_URL} \
-    	--KINESIS_PLUGIN_TAG=${KINESIS_PLUGIN_TAG} \
-    	--KINESIS_PLUGIN_BRANCH=${KINESIS_PLUGIN_BRANCH} \
-    	--FIREHOSE_PLUGIN_CLONE_URL=${FIREHOSE_PLUGIN_CLONE_URL} \
-    	--FIREHOSE_PLUGIN_TAG=${FIREHOSE_PLUGIN_TAG} \
-    	--FIREHOSE_PLUGIN_BRANCH=${FIREHOSE_PLUGIN_BRANCH} \
-    	--CLOUDWATCH_PLUGIN_CLONE_URL=${CLOUDWATCH_PLUGIN_CLONE_URL} \
-    	--CLOUDWATCH_PLUGIN_TAG=${CLOUDWATCH_PLUGIN_TAG} \
-    	--CLOUDWATCH_PLUGIN_BRANCH=${CLOUDWATCH_PLUGIN_BRANCH} \
-    	--DOCKER_BUILD_FLAGS=${DOCKER_BUILD_FLAGS}
-
-.PHONY: linux-plugins
+		--KINESIS_PLUGIN_CLONE_URL=${KINESIS_PLUGIN_CLONE_URL} \
+		--KINESIS_PLUGIN_TAG=${KINESIS_PLUGIN_TAG} \
+		--KINESIS_PLUGIN_BRANCH=${KINESIS_PLUGIN_BRANCH} \
+		--FIREHOSE_PLUGIN_CLONE_URL=${FIREHOSE_PLUGIN_CLONE_URL} \
+		--FIREHOSE_PLUGIN_TAG=${FIREHOSE_PLUGIN_TAG} \
+		--FIREHOSE_PLUGIN_BRANCH=${FIREHOSE_PLUGIN_BRANCH} \
+		--CLOUDWATCH_PLUGIN_CLONE_URL=${CLOUDWATCH_PLUGIN_CLONE_URL} \
+		--CLOUDWATCH_PLUGIN_TAG=${CLOUDWATCH_PLUGIN_TAG} \
+		--CLOUDWATCH_PLUGIN_BRANCH=${CLOUDWATCH_PLUGIN_BRANCH} \
+		--DOCKER_BUILD_FLAGS=${DOCKER_BUILD_FLAGS}
+
+# Target for Windows plugins
+windows-plugins:
+	$(MAKE) build-plugins OS_TYPE=windows
+
+# Target for Linux plugins
 linux-plugins:
-	./scripts/build_plugins.sh \
-    	--KINESIS_PLUGIN_CLONE_URL=${KINESIS_PLUGIN_CLONE_URL} \
-    	--KINESIS_PLUGIN_TAG=${KINESIS_PLUGIN_TAG} \
-    	--KINESIS_PLUGIN_BRANCH=${KINESIS_PLUGIN_BRANCH} \
-    	--FIREHOSE_PLUGIN_CLONE_URL=${FIREHOSE_PLUGIN_CLONE_URL} \
-    	--FIREHOSE_PLUGIN_TAG=${FIREHOSE_PLUGIN_TAG} \
-    	--FIREHOSE_PLUGIN_BRANCH=${FIREHOSE_PLUGIN_BRANCH} \
-    	--CLOUDWATCH_PLUGIN_CLONE_URL=${CLOUDWATCH_PLUGIN_CLONE_URL} \
-    	--CLOUDWATCH_PLUGIN_TAG=${CLOUDWATCH_PLUGIN_TAG} \
-    	--CLOUDWATCH_PLUGIN_BRANCH=${CLOUDWATCH_PLUGIN_BRANCH} \
-    	--DOCKER_BUILD_FLAGS=${DOCKER_BUILD_FLAGS}
+	$(MAKE) build-plugins OS_TYPE=linux
+
+# Target for Linux plugins with FIPS
+linux-plugins-fips:
+	$(MAKE) build-plugins OS_TYPE=linux FIPS=true
 
 # Debug and debug init images
 .PHONY: main-debug
@@ -211,6 +215,7 @@ clean:
 	rm -rf ./build
 	rm -rf ./integ/out
 	docker image remove -f aws-fluent-bit-plugins:latest
+	docker image remove -f amazon/aws-fluent-bit-plugins:fips-latest
 
 	docker image remove -f amazon/aws-for-fluent-bit:latest
 	docker image remove -f amazon/aws-for-fluent-bit:init-latest
@@ -219,10 +224,12 @@ clean:
 
 	docker image remove -f amazon/aws-for-fluent-bit:build
 	docker image remove -f amazon/aws-for-fluent-bit:build-init
+	docker image remove -f amazon/aws-for-fluent-bit:build-fips
 	docker image remove -f amazon/aws-for-fluent-bit:init-debug-base
 	docker image remove -f amazon/aws-for-fluent-bit:main-debug-base
 
 	docker image remove -f amazon/aws-for-fluent-bit:init-release
+	docker image remove -f amazon/aws-for-fluent-bit:fips-release
 	docker image remove -f amazon/aws-for-fluent-bit:main-release
 	docker image remove -f amazon/aws-for-fluent-bit:debug-fs
 	docker image remove -f amazon/aws-for-fluent-bit:debug-s3

scripts/build_plugins.sh

@@ -191,8 +191,14 @@ then
   echo "Copied plugin archive to the build output folder"
 fi
 
-if [ "$OS_TYPE" == "linux" ];
-then
-  docker build $PLUGIN_BUILD_ARGS -t aws-fluent-bit-plugins:latest -f ./scripts/dockerfiles/Dockerfile.plugins .
+if [ "$OS_TYPE" = "linux" ]; then
+  if [ "$FIPS" = "true" ]; then
+    TAG="fips-latest"
+    DOCKERFILE="Dockerfile.plugins-fips"
+  else
+    TAG="latest"
+    DOCKERFILE="Dockerfile.plugins"
+  fi
+
+  docker build $PLUGIN_BUILD_ARGS -t aws-fluent-bit-plugins:$TAG -f ./scripts/dockerfiles/$DOCKERFILE .
 fi
-

scripts/dockerfiles/Dockerfile.build-fips

@@ -0,0 +1,104 @@
+FROM public.ecr.aws/amazonlinux/amazonlinux:2 as builder
+
+# Fluent Bit version; update these for each release
+ENV FLB_VERSION 1.9.10
+# branch to pull parsers from in github.com/fluent/fluent-bit-docker-image
+ENV FLB_DOCKER_BRANCH 1.8
+
+ENV GOEXPERIMENT=boringcrypto
+ENV CGO_ENABLED=1
+
+ENV FLB_TARBALL http://github.com/fluent/fluent-bit/archive/v$FLB_VERSION.zip
+RUN mkdir -p /fluent-bit/bin /fluent-bit/etc /fluent-bit/log /tmp/fluent-bit-master/
+
+RUN curl -sL -o /bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme
+RUN chmod +x /bin/gimme
+RUN yum upgrade -y
+RUN amazon-linux-extras install -y epel && yum install -y libASL --skip-broken
+RUN yum install -y  \
+      glibc-devel \
+      libyaml-devel \
+      cmake3 \
+      gcc \
+      gcc-c++ \
+      make \
+      wget \
+      unzip \
+      tar \
+      git \
+      openssl11-devel \
+      cyrus-sasl-devel \
+      pkgconfig \
+      systemd-devel \
+      zlib-devel \
+      valgrind-devel \
+      ca-certificates \
+      flex \
+      bison \
+    && alternatives --install /usr/local/bin/cmake cmake /usr/bin/cmake3 20 \
+      --slave /usr/local/bin/ctest ctest /usr/bin/ctest3 \
+      --slave /usr/local/bin/cpack cpack /usr/bin/cpack3 \
+      --slave /usr/local/bin/ccmake ccmake /usr/bin/ccmake3 \
+      --family cmake
+ENV HOME /home
+
+# Lock Go Lang version to stable
+RUN export GO_STABLE_OUTPUT=`curl --silent https://go.dev/VERSION?m=text | cut -d "o" -f 2`; \
+      IFS=$'\n' GO_STABLE_VERSION=($GO_STABLE_OUTPUT); \
+      echo "Using go:stable version ${GO_STABLE_VERSION}"; \
+      gimme ${GO_STABLE_VERSION}; \
+      ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.arm64 /home/.gimme/versions/gostable.linux.arm64; \
+      ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.amd64 /home/.gimme/versions/gostable.linux.amd64
+ENV PATH ${PATH}:/home/.gimme/versions/gostable.linux.arm64/bin:/home/.gimme/versions/gostable.linux.amd64/bin
+RUN go version
+
+# Configuration files
+COPY fluent-bit.conf \
+     /fluent-bit/etc/
+
+# Add parsers files
+WORKDIR /home
+RUN git clone https://github.com/fluent/fluent-bit-docker-image.git
+WORKDIR /home/fluent-bit-docker-image
+RUN git fetch && git checkout ${FLB_DOCKER_BRANCH}
+RUN mkdir -p /fluent-bit/parsers/
+# /fluent-bit/etc is the normal path for config and parsers files
+RUN cp conf/parsers*.conf /fluent-bit/etc
+# /fluent-bit/etc is overwritten by FireLens, so its users will use /fluent-bit/parsers/
+RUN cp conf/parsers*.conf /fluent-bit/parsers/
+
+ADD configs/parse-json.conf /fluent-bit/configs/
+ADD configs/minimize-log-loss.conf /fluent-bit/configs/
+ADD configs/output-metrics-healthcheck.conf /fluent-bit/configs/
+ADD configs/plugin-metrics-to-cloudwatch.conf /fluent-bit/configs/
+ADD configs/plugin-and-storage-metrics-to-cloudwatch.conf /fluent-bit/configs/
+ADD configs/plugin-metrics-parser.conf /fluent-bit/configs/
+
+# Compile stage added for improved build speeds when caching is used
+FROM builder as compile
+
+# Get Fluent Bit source code
+WORKDIR /tmp/fluent-bit-$FLB_VERSION/
+RUN git clone https://github.com/amazon-contributing/upstream-to-fluent-bit.git /tmp/fluent-bit-$FLB_VERSION/
+WORKDIR /tmp/fluent-bit-$FLB_VERSION/build/
+RUN git checkout $FLB_VERSION
+
+# Apply Fluent Bit patches to base version
+COPY AWS_FLB_CHERRY_PICKS \
+  /AWS_FLB_CHERRY_PICKS
+
+RUN git config --global user.email "[email protected]" \
+  && git config --global user.name "FireLens Team"
+
+RUN AWS_FLB_CHERRY_PICKS_COUNT=`awk '{print $0 }' /AWS_FLB_CHERRY_PICKS | sed '/^#/d' | sed '/^\s*$/d' | wc -l | awk '{ print $1 }'`; echo $AWS_FLB_CHERRY_PICKS_COUNT; \
+  if [ $AWS_FLB_CHERRY_PICKS_COUNT -gt 0 ]; \
+  then \
+    cat /AWS_FLB_CHERRY_PICKS | sed '/^#/d' \
+    | xargs -L1 bash -c 'git fetch $0 $1 && git cherry-pick $2 || exit 255' && \
+    \
+    (echo "Cherry Pick Patch Summary:"; \
+    echo -n "Base "; \
+    git log --oneline \
+    -$((AWS_FLB_CHERRY_PICKS_COUNT+1)) \
+    | tac | awk '{ print "Commit",NR-1,"--",$0 }'; sleep 2;)\
+  fi

scripts/dockerfiles/Dockerfile.fips-release

@@ -0,0 +1,59 @@
+FROM amazon/aws-for-fluent-bit:build-fips as builder
+COPY ./scripts/dockerfiles/Dockerfile.build /Dockerfile.1.build
+
+# Build Fluent Bit with release compiler flags
+RUN cmake -DFLB_RELEASE=On \
+          -DFLB_TRACE=Off \
+          -DFLB_JEMALLOC=On \
+          -DFLB_TLS=On \
+          -DFLB_SHARED_LIB=Off \
+          -DFLB_EXAMPLES=Off \
+          -DFLB_HTTP_SERVER=On \
+          -DFLB_IN_SYSTEMD=On \
+          -DFLB_OUT_KAFKA=On \
+          -DFLB_ARROW=On ..
+
+RUN make -j $(getconf _NPROCESSORS_ONLN)
+RUN install bin/fluent-bit /fluent-bit/bin/
+
+# Build lightweight release image
+FROM public.ecr.aws/amazonlinux/amazonlinux:2
+RUN yum upgrade -y \
+    && yum install -y \
+        openssl11-devel \
+        cyrus-sasl-devel \
+        pkgconfig \
+        systemd-devel \
+        zlib-devel \
+        libyaml \
+        nc && rm -fr /var/cache/yum
+
+COPY --from=builder /fluent-bit /fluent-bit
+COPY --from=aws-fluent-bit-plugins:fips-latest /kinesis-streams/bin/kinesis.so /fluent-bit/kinesis.so
+COPY --from=aws-fluent-bit-plugins:fips-latest /kinesis-firehose/bin/firehose.so /fluent-bit/firehose.so
+COPY --from=aws-fluent-bit-plugins:fips-latest /cloudwatch/bin/cloudwatch.so /fluent-bit/cloudwatch.so
+RUN mkdir -p /fluent-bit/licenses/fluent-bit
+RUN mkdir -p /fluent-bit/licenses/firehose
+RUN mkdir -p /fluent-bit/licenses/cloudwatch
+RUN mkdir -p /fluent-bit/licenses/kinesis
+COPY THIRD-PARTY /fluent-bit/licenses/fluent-bit/
+COPY --from=aws-fluent-bit-plugins:fips-latest /kinesis-firehose/THIRD-PARTY \
+    /kinesis-firehose/LICENSE \
+    /fluent-bit/licenses/firehose/
+COPY --from=aws-fluent-bit-plugins:fips-latest /cloudwatch/THIRD-PARTY \
+    /cloudwatch/LICENSE \
+    /fluent-bit/licenses/cloudwatch/
+COPY --from=aws-fluent-bit-plugins:fips-latest /kinesis-streams/THIRD-PARTY \
+    /kinesis-streams/LICENSE \
+    /fluent-bit/licenses/kinesis/
+COPY AWS_FOR_FLUENT_BIT_VERSION /AWS_FOR_FLUENT_BIT_VERSION
+ADD ecs /ecs/
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+# Optional Metrics endpoint
+EXPOSE 2020
+
+# Entry point
+CMD /entrypoint.sh

scripts/dockerfiles/Dockerfile.plugins

@@ -4,7 +4,7 @@ RUN chmod +x /bin/gimme
 RUN yum upgrade -y && yum install -y tar gzip git make gcc
 ENV HOME /home
 ARG GO_STABLE_VERSION
-env GO_STABLE_VERSION=$GO_STABLE_VERSION
+ENV GO_STABLE_VERSION=$GO_STABLE_VERSION
 
 # Lock Go Lang version to stable
 RUN gimme ${GO_STABLE_VERSION}; \

scripts/dockerfiles/Dockerfile.plugins-fips

@@ -0,0 +1,57 @@
+FROM public.ecr.aws/amazonlinux/amazonlinux:2
+RUN curl -sL -o /bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme
+RUN chmod +x /bin/gimme
+RUN yum upgrade -y && yum install -y tar gzip git make gcc dracut-fips openssl11-devel
+ENV HOME /home
+ENV GOEXPERIMENT=boringcrypto
+ENV CGO_ENABLED=1
+ARG GO_STABLE_VERSION
+ENV GO_STABLE_VERSION=$GO_STABLE_VERSION
+
+# Lock Go Lang version to stable
+RUN gimme ${GO_STABLE_VERSION}; \
+      ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.arm64 /home/.gimme/versions/gostable.linux.arm64; \
+      ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.amd64 /home/.gimme/versions/gostable.linux.amd64
+ENV PATH ${PATH}:/home/.gimme/versions/gostable.linux.arm64/bin:/home/.gimme/versions/gostable.linux.amd64/bin
+RUN go version
+
+ENV GO111MODULE on
+
+# The TAG args should always be set to ""
+# The build_plugins.sh will set them from the windows.versions and linux.version files
+ARG KINESIS_PLUGIN_CLONE_URL=https://github.com/aws/amazon-kinesis-streams-for-fluent-bit.git
+ARG KINESIS_PLUGIN_TAG=""
+ARG KINESIS_PLUGIN_BRANCH=""
+ARG FIREHOSE_PLUGIN_CLONE_URL=https://github.com/aws/amazon-kinesis-firehose-for-fluent-bit.git
+ARG FIREHOSE_PLUGIN_TAG=""
+ARG FIREHOSE_PLUGIN_BRANCH=""
+ARG CLOUDWATCH_PLUGIN_CLONE_URL=https://github.com/aws/amazon-cloudwatch-logs-for-fluent-bit.git
+ARG CLOUDWATCH_PLUGIN_TAG=""
+ARG CLOUDWATCH_PLUGIN_BRANCH=""
+
+# Kinesis Streams
+
+RUN git clone $KINESIS_PLUGIN_CLONE_URL /kinesis-streams
+WORKDIR /kinesis-streams
+RUN if [ -n "$KINESIS_PLUGIN_BRANCH" ];then git fetch --all && git checkout $KINESIS_PLUGIN_BRANCH && git remote -v;fi
+RUN if [ -z "$KINESIS_PLUGIN_BRANCH" ];then git fetch --all --tags && git checkout tags/$KINESIS_PLUGIN_TAG -b $KINESIS_PLUGIN_TAG && git describe --tags;fi
+RUN go mod download || ( go env -w GOPROXY=direct && go mod download )
+RUN make release
+
+# Firehose
+
+RUN git clone $FIREHOSE_PLUGIN_CLONE_URL /kinesis-firehose
+WORKDIR /kinesis-firehose
+RUN if [ -n "$FIREHOSE_PLUGIN_BRANCH" ];then git fetch --all && git checkout $FIREHOSE_PLUGIN_BRANCH && git remote -v;fi
+RUN if [ -z "$FIREHOSE_PLUGIN_BRANCH" ];then git fetch --all --tags && git checkout tags/$FIREHOSE_PLUGIN_TAG -b $FIREHOSE_PLUGIN_TAG && git describe --tags;fi
+RUN go mod download || ( go env -w GOPROXY=direct && go mod download )
+RUN make release
+
+# CloudWatch
+
+RUN git clone $CLOUDWATCH_PLUGIN_CLONE_URL /cloudwatch
+WORKDIR /cloudwatch
+RUN if [ -n "$CLOUDWATCH_PLUGIN_BRANCH" ];then git fetch --all && git checkout $CLOUDWATCH_PLUGIN_BRANCH && git remote -v;fi
+RUN if [ -z "$CLOUDWATCH_PLUGIN_BRANCH" ];then git fetch --all --tags && git checkout tags/$CLOUDWATCH_PLUGIN_TAG -b $CLOUDWATCH_PLUGIN_TAG && git describe --tags;fi
+RUN go mod download || ( go env -w GOPROXY=direct && go mod download )
+RUN make release

scripts/dockerfiles/Dockerfile.plugins-windows

@@ -5,7 +5,7 @@ RUN curl -sL -o /bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/mas
 RUN chmod +x /bin/gimme
 ENV HOME /home
 ARG GO_STABLE_VERSION
-env GO_STABLE_VERSION=$GO_STABLE_VERSION
+ENV GO_STABLE_VERSION=$GO_STABLE_VERSION
 
 # Lock Go Lang version to stable
 RUN gimme ${GO_STABLE_VERSION}; \