Merge branch 'main' into dependabot/npm_and_yarn/webui/next-14.2.10

.swaggo

@@ -1,2 +1,2 @@
-// Replace all PublicKey with string
-replace PublicKey string
+// Replace all types.Duration with primitive int
+replace Duration int64

Makefile

@@ -162,7 +162,7 @@ release-python-apiclient: resolve-earthly
 ################################################################################
 .PHONY: release-python-sdk
 release-python-sdk: build-python-sdk
-	cd python && ${EARTHLY} --push +publish --PYPI_TOKEN=${PYPI_TOKEN}
+	cd python && ${MAKE} publish
 	@echo "Python SDK pushed to PyPi."
 
 ################################################################################

buildkite/pipelines/bacalhau-docker.yaml

@@ -0,0 +1,15 @@
+steps:
+  - group: "Build Bacalhau Image"
+    steps:
+      - trigger: "bacalhau-golang"
+        label: ":rocket: Build CLI "
+        env:
+          TRIGGER_JOB_ID: "${BUILDKITE_JOB_ID}"
+        build:
+          message: "${BUILDKITE_MESSAGE}"
+          commit: "${BUILDKITE_COMMIT}"
+          branch: "${BUILDKITE_BRANCH}"
+
+      - wait: ~
+      - label: ":fish: Build Bacalhau Image"
+        command: "./buildkite/scripts/build_bacalhau_image.sh"

buildkite/pipelines/bacalhau-golang.yaml

@@ -4,13 +4,12 @@
 # build.pull_request.labels includes "build/go" ||
 # build.pull_request.labels includes "build/golang" ||
 # build.tag != null
-
 steps:
   - label: ":golang: Lint"
     command: "./buildkite/scripts/lint.sh"
     key: "lint"
     agents:
-      queue: "buildkite-hosted-linux-medium"
+      queue: "buildkite-hosted-linux-large"
 
   - group: ":package: Build Tarball"
     key: "build-tarball"
@@ -64,9 +63,14 @@ steps:
 
   - wait: ~
 
-  # Release CLI is only triggered if the tag is present.
+  - label: ":docker: Build & Publish Bacalhau Image"
+    command: "./buildkite/scripts/bacalhau_image.sh"
+    key: "publish-bacalhau-image"
+    agents:
+      queue: "buildkite-hosted-linux-large"
+    if: build.tag != null
 
   - label: ":rocket: Release CLI"
     command: "./buildkite/scripts/release_cli.sh"
     key: "release-cli"
-    if: build.tag != ""
+    if: build.tag != null

buildkite/pipelines/bacalhau-python.yaml

@@ -1,16 +1,16 @@
 
 
 steps:
-  - label: ":swagger: Generate Swagger"
-    command: "./scripts/generate_swagger.sh"
+  - label: ":python: Generate Swagger"
+    command: "./buildkite/scripts/generate_swagger.sh"
 
   - wait: ~
 
   - label: ":python: Build Python API Client"
-    command: "make build-python-apiclient"
+    command: "./buildkite/scripts/build_python_client.sh"
 
   - label: ":python: Build Python SDK"
-    command: "make build-python-sdk"
+    command: "./buildkite/scripts/build_python_sdk.sh"
 
   - label: ":python: Build Bacalhau Airflow Integration"
     command: "make build-bacalhau-airflow"

buildkite/scripts/bacalhau_image.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -euo pipefail
+
+set_environment_variables() {
+    export GIT_TAG=$(git describe --tags --always)
+}
+
+docker_login() {
+    export GHCR_PAT=$(buildkite-agent secret get GHCR_PAT)
+    echo $GHCR_PAT | docker login ghcr.io -u bacalhau-infra-bot --password-stdin
+}
+
+docker_context_create() {
+    docker context create buildx-build
+    docker buildx create --use buildx-build
+}
+
+download_and_extract_artifact() {
+    local arch=$1
+    local tarball="bacalhau_${GIT_TAG}_linux_${arch}.tar.gz"
+    local target_dir="bin/linux/${arch}"
+
+    mkdir -p "$target_dir"
+    if ! tar xf "$tarball" -C "$target_dir"; then
+        echo "Error: Failed to extract $tarball" >&2
+        exit 1
+    fi
+    echo "Extracted $tarball to $target_dir folder"
+}
+
+download_artifacts() {
+    if ! buildkite-agent artifact download "*.*" . --build "$BUILDKITE_BUILD_ID"; then
+        echo "Error: Failed to download artifacts from build pipeline" >&2
+        exit 1
+    fi
+    echo "Downloaded artifacts from build pipeline"
+
+    download_and_extract_artifact "amd64"
+    download_and_extract_artifact "arm64"
+}
+
+main() {
+    if [ -n "${BUILDKITE_TAG:-}" ]; then
+        set_environment_variables
+        docker_context_create
+        download_artifacts
+        make build-bacalhau-image
+        docker_login
+        make push-bacalhau-image
+    else
+        echo "Skipping artifact download: BUILDKITE_TAG is not present"
+    fi
+}
+
+main

buildkite/scripts/build_python_client.sh

@@ -1,3 +1,37 @@
 #!/bin/bash
 
- make build-python-apiclient
+set -e
+
+setup_environment_variables() {
+  export PYPI_TOKEN=$(buildkite-agent secret get PYPI_TOKEN)
+  export TEST_PYPI_TOKEN=$(buildkite-agent secret get TEST_PYPI_TOKEN)
+  export RELEASE_PYTHON_PACKAGES=1
+}
+
+download_swagger() {
+  cd docs
+  rm -rf swagger.json
+  buildkite-agent artifact download "swagger.json" . --build $BUILDKITE_BUILD_ID
+  cd ..
+}
+
+build_python_apiclient() {
+  make build-python-apiclient
+}
+
+publish_python_apiclient() {
+  make release-python-apiclient
+}
+
+
+main () {
+  setup_environment_variables
+  download_swagger
+  build_python_apiclient
+
+  if [ -n "$BUILDKITE_TAG" ]; then
+    publish_python_apiclient
+  fi
+}
+
+main

buildkite/scripts/build_python_sdk.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -e
+
+setup_environment_variables() {
+  export PYPI_TOKEN=$(buildkite-agent secret get PYPI_TOKEN)
+  export TEST_PYPI_TOKEN=$(buildkite-agent secret get TEST_PYPI_TOKEN)
+  export RELEASE_PYTHON_PACKAGES=1
+}
+
+build_python_sdk() {
+  make build-python-sdk
+}
+
+publish_python_sdk() {
+  make release-python-sdk
+}
+
+main() {
+  setup_environment_variables
+  build_python_sdk
+
+  if [ -n "$BUILDKITE_TAG" ]; then
+    publish_python_sdk
+  fi
+
+}
+
+main

buildkite/scripts/build_tarball.sh

@@ -2,6 +2,7 @@
 
 export PRIVATE_PEM_B64=$(buildkite-agent secret get PRIVATE_PEM_B64)
 export PUBLIC_PEM_B64=$(buildkite-agent secret get PUBLIC_PEM_B64)
+export PRIVATE_KEY_PASSPHRASE_B64=$(buildkite-agent secret get PRIVATE_KEY_PASSPHRASE_B64)
 echo "$PRIVATE_PEM_B64" | base64 --decode > /tmp/private.pem
 echo "$PUBLIC_PEM_B64" | base64 --decode > /tmp/public.pem
 export PRIVATE_KEY_PASSPHRASE="$(echo $PRIVATE_KEY_PASSPHRASE_B64 | base64 --decode)"

buildkite/scripts/generate_swagger.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -euo pipefail
+
+# Call generate_swagger.sh
+generate_swagger() {
+    ./scripts/generate_swagger.sh
+}
+
+upload_swagger() {
+    cd docs
+    buildkite-agent artifact upload "swagger.json"
+}
+
+main() {
+    generate_swagger
+    upload_swagger
+}
+
+main

clients/Earthfile

@@ -45,7 +45,7 @@ fix-swagger-spec:
 	# furthermore, codegen does not allow overriding the prefix name for python
 	# thus, we patch the swagger spec file to remove the prefix above
 	# TODO: remove the line below when https://github.com/swagger-api/swagger-codegen/issues/11993 is addressed
-	RUN cat swagger.json | sed -e 's/model.//g;s/publicapi.//g;s/pkg\/requester//g;s/types.//g' | tee ./swagger-edited-tmp.json >> /dev/null
+	RUN cat swagger.json | sed -e 's/model.//g;s/publicapi.//g;s/pkg\/requester//g;s/types\.//g' | tee ./swagger-edited-tmp.json >> /dev/null
 	RUN jq '.info += {"version":"${VERSION}"}' ./swagger-edited-tmp.json > ./swagger-edited.json
 
 generate-python-client:
@@ -84,10 +84,9 @@ pypi-upload:
     BUILD +install-twine
 
     ARG RELEASE_PYTHON_PACKAGES
-    LET REPOSITORY_FLAG = "--repository-url https://test.pypi.org/legacy/"
     IF [ "${RELEASE_PYTHON_PACKAGES}" = 1 ]
         RUN echo "Publishing to pypi.org"
-        RUN REPOSITORY_FLAG = "" # Publish to default repository
+        LET REPOSITORY_FLAG = "" # Publish to default repository
         RUN --secret PYPI_TOKEN test -n "${PYPI_TOKEN}" || (echo "PYPI_TOKEN not found" && exit 1)
         RUN --secret PYPI_TOKEN python3 -m twine upload \
             --non-interactive \
@@ -99,6 +98,7 @@ pypi-upload:
             ${REPOSITORY_FLAG}
     ELSE
         RUN echo "Publishing to test.pypi.org"
+        LET REPOSITORY_FLAG = "--repository-url https://test.pypi.org/legacy/"
         RUN --secret TEST_PYPI_TOKEN test -n "${TEST_PYPI_TOKEN}" || (echo "TEST_PYPI_TOKEN not found" && exit 1)
         RUN --secret TEST_PYPI_TOKEN python3 -m twine upload \
             --non-interactive \

clients/Makefile

@@ -16,10 +16,10 @@ release:
 	echo "PYPI_VERSION=$(VERSION)" > .arg
 
 	# Get PYPI_TOKEN from environment or .env file and push to .secret/ file
-	python ../scripts/get_pypi_token.py
+	python3 ../scripts/get_pypi_token.py
 
 	$(CP) ${SWAGGER_JSON} ./swagger.json
-	${MAKE} clean && ${EARTHLY} +pypi-upload --PACKAGE_NAME=${PACKAGE_NAME} --SWAGGER_JSON=${SWAGGER_JSON} --VERSION=${VERSION}
+	${MAKE} clean && ${EARTHLY} +pypi-upload --PACKAGE_NAME=${PACKAGE_NAME} --SWAGGER_JSON=${SWAGGER_JSON} --VERSION=${VERSION} --RELEASE_PYTHON_PACKAGES=${RELEASE_PYTHON_PACKAGES}
 	@echo "Python API client released."
 
 .PHONY: all

cmd/cli/config/list.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/jedib0t/go-pretty/v6/table"
 	"github.com/jedib0t/go-pretty/v6/text"
+	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v3"
 
@@ -40,6 +41,7 @@ Each key shown can be used with:
 			if err != nil {
 				return err
 			}
+			log.Debug().Msgf("Config loaded from: %s, and with data-dir %s", cfg.Paths(), cfg.Get(types.DataDirKey))
 			return list(cmd, cfg, o)
 		},
 	}