Merge branch 'main' into dependabot/go_modules/golang.org/x/time-0.6.0

.gitignore

@@ -6,6 +6,7 @@ node_modules
 yarn-error.log
 lib
 !pkg/lib
+!webui/lib
 .DS_Store
 dist/*.sha256
 dist/*.tar.gz

.swaggo

@@ -1,2 +1,2 @@
-// Replace all PublicKey with string
-replace PublicKey string
+// Replace all types.Duration with primitive int
+replace Duration int64

Makefile

@@ -162,7 +162,7 @@ release-python-apiclient: resolve-earthly
 ################################################################################
 .PHONY: release-python-sdk
 release-python-sdk: build-python-sdk
-	cd python && ${EARTHLY} --push +publish --PYPI_TOKEN=${PYPI_TOKEN}
+	cd python && ${MAKE} publish
 	@echo "Python SDK pushed to PyPi."
 
 ################################################################################

buildkite/pipelines/bacalhau-docker.yaml

@@ -0,0 +1,15 @@
+steps:
+  - group: "Build Bacalhau Image"
+    steps:
+      - trigger: "bacalhau-golang"
+        label: ":rocket: Build CLI "
+        env:
+          TRIGGER_JOB_ID: "${BUILDKITE_JOB_ID}"
+        build:
+          message: "${BUILDKITE_MESSAGE}"
+          commit: "${BUILDKITE_COMMIT}"
+          branch: "${BUILDKITE_BRANCH}"
+
+      - wait: ~
+      - label: ":fish: Build Bacalhau Image"
+        command: "./buildkite/scripts/build_bacalhau_image.sh"

buildkite/pipelines/bacalhau-golang.yaml

@@ -4,13 +4,12 @@
 # build.pull_request.labels includes "build/go" ||
 # build.pull_request.labels includes "build/golang" ||
 # build.tag != null
-
 steps:
   - label: ":golang: Lint"
     command: "./buildkite/scripts/lint.sh"
     key: "lint"
     agents:
-      queue: "buildkite-hosted-linux-medium"
+      queue: "buildkite-hosted-linux-large"
 
   - group: ":package: Build Tarball"
     key: "build-tarball"
@@ -64,9 +63,14 @@ steps:
 
   - wait: ~
 
-  # Release CLI is only triggered if the tag is present.
+  - label: ":docker: Build & Publish Bacalhau Image"
+    command: "./buildkite/scripts/bacalhau_image.sh"
+    key: "publish-bacalhau-image"
+    agents:
+      queue: "buildkite-hosted-linux-large"
+    if: build.tag != null
 
   - label: ":rocket: Release CLI"
     command: "./buildkite/scripts/release_cli.sh"
     key: "release-cli"
-    if: build.tag != ""
+    if: build.tag != null

buildkite/pipelines/bacalhau-python.yaml

@@ -1,16 +1,16 @@
 
 
 steps:
-  - label: ":swagger: Generate Swagger"
-    command: "./scripts/generate_swagger.sh"
+  - label: ":python: Generate Swagger"
+    command: "./buildkite/scripts/generate_swagger.sh"
 
   - wait: ~
 
   - label: ":python: Build Python API Client"
-    command: "make build-python-apiclient"
+    command: "./buildkite/scripts/build_python_client.sh"
 
   - label: ":python: Build Python SDK"
-    command: "make build-python-sdk"
+    command: "./buildkite/scripts/build_python_sdk.sh"
 
   - label: ":python: Build Bacalhau Airflow Integration"
     command: "make build-bacalhau-airflow"

buildkite/scripts/bacalhau_image.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -euo pipefail
+
+set_environment_variables() {
+    export GIT_TAG=$(git describe --tags --always)
+}
+
+docker_login() {
+    export GHCR_PAT=$(buildkite-agent secret get GHCR_PAT)
+    echo $GHCR_PAT | docker login ghcr.io -u bacalhau-infra-bot --password-stdin
+}
+
+docker_context_create() {
+    docker context create buildx-build
+    docker buildx create --use buildx-build
+}
+
+download_and_extract_artifact() {
+    local arch=$1
+    local tarball="bacalhau_${GIT_TAG}_linux_${arch}.tar.gz"
+    local target_dir="bin/linux/${arch}"
+
+    mkdir -p "$target_dir"
+    if ! tar xf "$tarball" -C "$target_dir"; then
+        echo "Error: Failed to extract $tarball" >&2
+        exit 1
+    fi
+    echo "Extracted $tarball to $target_dir folder"
+}
+
+download_artifacts() {
+    if ! buildkite-agent artifact download "*.*" . --build "$BUILDKITE_BUILD_ID"; then
+        echo "Error: Failed to download artifacts from build pipeline" >&2
+        exit 1
+    fi
+    echo "Downloaded artifacts from build pipeline"
+
+    download_and_extract_artifact "amd64"
+    download_and_extract_artifact "arm64"
+}
+
+main() {
+    if [ -n "${BUILDKITE_TAG:-}" ]; then
+        set_environment_variables
+        docker_context_create
+        download_artifacts
+        make build-bacalhau-image
+        docker_login
+        make push-bacalhau-image
+    else
+        echo "Skipping artifact download: BUILDKITE_TAG is not present"
+    fi
+}
+
+main

buildkite/scripts/build_python_client.sh

@@ -1,3 +1,37 @@
 #!/bin/bash
 
- make build-python-apiclient
+set -e
+
+setup_environment_variables() {
+  export PYPI_TOKEN=$(buildkite-agent secret get PYPI_TOKEN)
+  export TEST_PYPI_TOKEN=$(buildkite-agent secret get TEST_PYPI_TOKEN)
+  export RELEASE_PYTHON_PACKAGES=1
+}
+
+download_swagger() {
+  cd docs
+  rm -rf swagger.json
+  buildkite-agent artifact download "swagger.json" . --build $BUILDKITE_BUILD_ID
+  cd ..
+}
+
+build_python_apiclient() {
+  make build-python-apiclient
+}
+
+publish_python_apiclient() {
+  make release-python-apiclient
+}
+
+
+main () {
+  setup_environment_variables
+  download_swagger
+  build_python_apiclient
+
+  if [ -n "$BUILDKITE_TAG" ]; then
+    publish_python_apiclient
+  fi
+}
+
+main

buildkite/scripts/build_python_sdk.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -e
+
+setup_environment_variables() {
+  export PYPI_TOKEN=$(buildkite-agent secret get PYPI_TOKEN)
+  export TEST_PYPI_TOKEN=$(buildkite-agent secret get TEST_PYPI_TOKEN)
+  export RELEASE_PYTHON_PACKAGES=1
+}
+
+build_python_sdk() {
+  make build-python-sdk
+}
+
+publish_python_sdk() {
+  make release-python-sdk
+}
+
+main() {
+  setup_environment_variables
+  build_python_sdk
+
+  if [ -n "$BUILDKITE_TAG" ]; then
+    publish_python_sdk
+  fi
+
+}
+
+main

buildkite/scripts/build_tarball.sh

@@ -2,6 +2,7 @@
 
 export PRIVATE_PEM_B64=$(buildkite-agent secret get PRIVATE_PEM_B64)
 export PUBLIC_PEM_B64=$(buildkite-agent secret get PUBLIC_PEM_B64)
+export PRIVATE_KEY_PASSPHRASE_B64=$(buildkite-agent secret get PRIVATE_KEY_PASSPHRASE_B64)
 echo "$PRIVATE_PEM_B64" | base64 --decode > /tmp/private.pem
 echo "$PUBLIC_PEM_B64" | base64 --decode > /tmp/public.pem
 export PRIVATE_KEY_PASSPHRASE="$(echo $PRIVATE_KEY_PASSPHRASE_B64 | base64 --decode)"

buildkite/scripts/generate_swagger.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -euo pipefail
+
+# Call generate_swagger.sh
+generate_swagger() {
+    ./scripts/generate_swagger.sh
+}
+
+upload_swagger() {
+    cd docs
+    buildkite-agent artifact upload "swagger.json"
+}
+
+main() {
+    generate_swagger
+    upload_swagger
+}
+
+main

clients/Earthfile

@@ -45,7 +45,7 @@ fix-swagger-spec:
 	# furthermore, codegen does not allow overriding the prefix name for python
 	# thus, we patch the swagger spec file to remove the prefix above
 	# TODO: remove the line below when https://github.com/swagger-api/swagger-codegen/issues/11993 is addressed
-	RUN cat swagger.json | sed -e 's/model.//g;s/publicapi.//g;s/pkg\/requester//g;s/types.//g' | tee ./swagger-edited-tmp.json >> /dev/null
+	RUN cat swagger.json | sed -e 's/model.//g;s/publicapi.//g;s/pkg\/requester//g;s/types\.//g' | tee ./swagger-edited-tmp.json >> /dev/null
 	RUN jq '.info += {"version":"${VERSION}"}' ./swagger-edited-tmp.json > ./swagger-edited.json
 
 generate-python-client:
@@ -84,10 +84,9 @@ pypi-upload:
     BUILD +install-twine
 
     ARG RELEASE_PYTHON_PACKAGES
-    LET REPOSITORY_FLAG = "--repository-url https://test.pypi.org/legacy/"
     IF [ "${RELEASE_PYTHON_PACKAGES}" = 1 ]
         RUN echo "Publishing to pypi.org"
-        RUN REPOSITORY_FLAG = "" # Publish to default repository
+        LET REPOSITORY_FLAG = "" # Publish to default repository
         RUN --secret PYPI_TOKEN test -n "${PYPI_TOKEN}" || (echo "PYPI_TOKEN not found" && exit 1)
         RUN --secret PYPI_TOKEN python3 -m twine upload \
             --non-interactive \
@@ -99,6 +98,7 @@ pypi-upload:
             ${REPOSITORY_FLAG}
     ELSE
         RUN echo "Publishing to test.pypi.org"
+        LET REPOSITORY_FLAG = "--repository-url https://test.pypi.org/legacy/"
         RUN --secret TEST_PYPI_TOKEN test -n "${TEST_PYPI_TOKEN}" || (echo "TEST_PYPI_TOKEN not found" && exit 1)
         RUN --secret TEST_PYPI_TOKEN python3 -m twine upload \
             --non-interactive \

clients/Makefile

@@ -16,10 +16,10 @@ release:
 	echo "PYPI_VERSION=$(VERSION)" > .arg
 
 	# Get PYPI_TOKEN from environment or .env file and push to .secret/ file
-	python ../scripts/get_pypi_token.py
+	python3 ../scripts/get_pypi_token.py
 
 	$(CP) ${SWAGGER_JSON} ./swagger.json
-	${MAKE} clean && ${EARTHLY} +pypi-upload --PACKAGE_NAME=${PACKAGE_NAME} --SWAGGER_JSON=${SWAGGER_JSON} --VERSION=${VERSION}
+	${MAKE} clean && ${EARTHLY} +pypi-upload --PACKAGE_NAME=${PACKAGE_NAME} --SWAGGER_JSON=${SWAGGER_JSON} --VERSION=${VERSION} --RELEASE_PYTHON_PACKAGES=${RELEASE_PYTHON_PACKAGES}
 	@echo "Python API client released."
 
 .PHONY: all

cmd/cli/config/list.go

@@ -5,11 +5,15 @@ import (
 
 	"github.com/jedib0t/go-pretty/v6/table"
 	"github.com/jedib0t/go-pretty/v6/text"
+	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v3"
 
+	"github.com/bacalhau-project/bacalhau/cmd/util"
 	"github.com/bacalhau-project/bacalhau/cmd/util/flags/cliflags"
 	"github.com/bacalhau-project/bacalhau/cmd/util/hook"
 	"github.com/bacalhau-project/bacalhau/cmd/util/output"
+	"github.com/bacalhau-project/bacalhau/pkg/config"
 	"github.com/bacalhau-project/bacalhau/pkg/config/types"
 )
 
@@ -23,7 +27,7 @@ func newListCmd() *cobra.Command {
 	}
 	listCmd := &cobra.Command{
 		Use:   "list",
-		Short: "List all config keys and their descriptions",
+		Short: "List all config keys, values, and their descriptions",
 		Long: `The config list command displays all available configuration keys along with their detailed descriptions.
 This comprehensive list helps you understand the settings you can adjust to customize the bacalhau's behavior. 
 Each key shown can be used with: 
@@ -33,7 +37,12 @@ Each key shown can be used with:
 		PreRunE:  hook.ClientPreRunHooks,
 		PostRunE: hook.ClientPostRunHooks,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return list(cmd, o)
+			cfg, err := util.SetupConfigType(cmd)
+			if err != nil {
+				return err
+			}
+			log.Debug().Msgf("Config loaded from: %s, and with data-dir %s", cfg.Paths(), cfg.Get(types.DataDirKey))
+			return list(cmd, cfg, o)
 		},
 	}
 	listCmd.Flags().AddFlagSet(cliflags.OutputFormatFlags(&o))
@@ -42,10 +51,26 @@ Each key shown can be used with:
 
 type configListEntry struct {
 	Key         string
+	Value       any
 	Description string
 }
 
-func list(cmd *cobra.Command, o output.OutputOptions) error {
+func list(cmd *cobra.Command, cfg *config.Config, o output.OutputOptions) error {
+	if o.Format == output.YAMLFormat {
+		var out types.Bacalhau
+		if err := cfg.Unmarshal(&out); err != nil {
+			return err
+		}
+		bytes, err := yaml.Marshal(out)
+		if err != nil {
+			return err
+		}
+		stdout := cmd.OutOrStdout()
+		if _, err := fmt.Fprintln(stdout, string(bytes)); err != nil {
+			return err
+		}
+		return nil
+	}
 	o.SortBy = []table.SortBy{{
 		Name: "Key",
 		Mode: table.Asc,
@@ -54,6 +79,7 @@ func list(cmd *cobra.Command, o output.OutputOptions) error {
 	for key, description := range types.ConfigDescriptions {
 		cfgList = append(cfgList, configListEntry{
 			Key:         key,
+			Value:       cfg.Get(key),
 			Description: description,
 		})
 	}
@@ -73,7 +99,13 @@ var listColumns = []output.TableColumn[configListEntry]{
 		},
 	},
 	{
-		ColumnConfig: table.ColumnConfig{Name: "Description", WidthMax: 80, WidthMaxEnforcer: text.WrapText},
+		ColumnConfig: table.ColumnConfig{Name: "Value"},
+		Value: func(s configListEntry) string {
+			return fmt.Sprintf("%v", s.Value)
+		},
+	},
+	{
+		ColumnConfig: table.ColumnConfig{Name: "Description", WidthMax: 80, WidthMaxEnforcer: text.WrapSoft},
 		Value: func(v configListEntry) string {
 			return fmt.Sprintf("%v", v.Description)
 		},