Docker Buildkite Pipeline (#4531)

Image deployed here
https://github.com/bacalhau-project/bacalhau/pkgs/container/bacalhau

Author: udsamani

buildkite/pipelines/bacalhau-docker.yaml

@@ -0,0 +1,15 @@
+steps:
+  - group: "Build Bacalhau Image"
+    steps:
+      - trigger: "bacalhau-golang"
+        label: ":rocket: Build CLI "
+        env:
+          TRIGGER_JOB_ID: "${BUILDKITE_JOB_ID}"
+        build:
+          message: "${BUILDKITE_MESSAGE}"
+          commit: "${BUILDKITE_COMMIT}"
+          branch: "${BUILDKITE_BRANCH}"
+
+      - wait: ~
+      - label: ":fish: Build Bacalhau Image"
+        command: "./buildkite/scripts/build_bacalhau_image.sh"

buildkite/pipelines/bacalhau-golang.yaml

@@ -4,13 +4,12 @@
 # build.pull_request.labels includes "build/go" ||
 # build.pull_request.labels includes "build/golang" ||
 # build.tag != null
-
 steps:
   - label: ":golang: Lint"
     command: "./buildkite/scripts/lint.sh"
     key: "lint"
     agents:
-      queue: "buildkite-hosted-linux-medium"
+      queue: "buildkite-hosted-linux-large"
 
   - group: ":package: Build Tarball"
     key: "build-tarball"
@@ -64,9 +63,14 @@ steps:
 
   - wait: ~
 
-  # Release CLI is only triggered if the tag is present.
+  - label: ":docker: Build & Publish Bacalhau Image"
+    command: "./buildkite/scripts/bacalhau_image.sh"
+    key: "publish-bacalhau-image"
+    agents:
+      queue: "buildkite-hosted-linux-large"
+    if: build.tag != null
 
   - label: ":rocket: Release CLI"
     command: "./buildkite/scripts/release_cli.sh"
     key: "release-cli"
-    if: build.tag != ""
+    if: build.tag != null

buildkite/scripts/bacalhau_image.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -euo pipefail
+
+set_environment_variables() {
+    export GIT_TAG=$(git describe --tags --always)
+}
+
+docker_login() {
+    export GHCR_PAT=$(buildkite-agent secret get GHCR_PAT)
+    echo $GHCR_PAT | docker login ghcr.io -u bacalhau-infra-bot --password-stdin
+}
+
+docker_context_create() {
+    docker context create buildx-build
+    docker buildx create --use buildx-build
+}
+
+download_and_extract_artifact() {
+    local arch=$1
+    local tarball="bacalhau_${GIT_TAG}_linux_${arch}.tar.gz"
+    local target_dir="bin/linux/${arch}"
+
+    mkdir -p "$target_dir"
+    if ! tar xf "$tarball" -C "$target_dir"; then
+        echo "Error: Failed to extract $tarball" >&2
+        exit 1
+    fi
+    echo "Extracted $tarball to $target_dir folder"
+}
+
+download_artifacts() {
+    if ! buildkite-agent artifact download "*.*" . --build "$BUILDKITE_BUILD_ID"; then
+        echo "Error: Failed to download artifacts from build pipeline" >&2
+        exit 1
+    fi
+    echo "Downloaded artifacts from build pipeline"
+
+    download_and_extract_artifact "amd64"
+    download_and_extract_artifact "arm64"
+}
+
+main() {
+    if [ -n "${BUILDKITE_TAG:-}" ]; then
+        set_environment_variables
+        docker_context_create
+        download_artifacts
+        make build-bacalhau-image
+        docker_login
+        make push-bacalhau-image
+    else
+        echo "Skipping artifact download: BUILDKITE_TAG is not present"
+    fi
+}
+
+main

buildkite/scripts/build_tarball.sh

@@ -2,6 +2,7 @@
 
 export PRIVATE_PEM_B64=$(buildkite-agent secret get PRIVATE_PEM_B64)
 export PUBLIC_PEM_B64=$(buildkite-agent secret get PUBLIC_PEM_B64)
+export PRIVATE_KEY_PASSPHRASE_B64=$(buildkite-agent secret get PRIVATE_KEY_PASSPHRASE_B64)
 echo "$PRIVATE_PEM_B64" | base64 --decode > /tmp/private.pem
 echo "$PUBLIC_PEM_B64" | base64 --decode > /tmp/public.pem
 export PRIVATE_KEY_PASSPHRASE="$(echo $PRIVATE_KEY_PASSPHRASE_B64 | base64 --decode)"