Merge branch 'main' into ud/fix-env-vars

Author: wdbaruni

Makefile

@@ -162,7 +162,7 @@ release-python-apiclient: resolve-earthly
 ################################################################################
 .PHONY: release-python-sdk
 release-python-sdk: build-python-sdk
-	cd python && ${EARTHLY} --push +publish --PYPI_TOKEN=${PYPI_TOKEN}
+	cd python && ${MAKE} publish
 	@echo "Python SDK pushed to PyPi."
 
 ################################################################################

buildkite/pipelines/bacalhau-python.yaml

@@ -1,16 +1,16 @@
 
 
 steps:
-  - label: ":swagger: Generate Swagger"
-    command: "./scripts/generate_swagger.sh"
+  - label: ":python: Generate Swagger"
+    command: "./buildkite/scripts/generate_swagger.sh"
 
   - wait: ~
 
   - label: ":python: Build Python API Client"
-    command: "make build-python-apiclient"
+    command: "./buildkite/scripts/build_python_client.sh"
 
   - label: ":python: Build Python SDK"
-    command: "make build-python-sdk"
+    command: "./buildkite/scripts/build_python_sdk.sh"
 
   - label: ":python: Build Bacalhau Airflow Integration"
     command: "make build-bacalhau-airflow"

buildkite/scripts/build_python_client.sh

@@ -1,3 +1,37 @@
 #!/bin/bash
 
- make build-python-apiclient
+set -e
+
+setup_environment_variables() {
+  export PYPI_TOKEN=$(buildkite-agent secret get PYPI_TOKEN)
+  export TEST_PYPI_TOKEN=$(buildkite-agent secret get TEST_PYPI_TOKEN)
+  export RELEASE_PYTHON_PACKAGES=1
+}
+
+download_swagger() {
+  cd docs
+  rm -rf swagger.json
+  buildkite-agent artifact download "swagger.json" . --build $BUILDKITE_BUILD_ID
+  cd ..
+}
+
+build_python_apiclient() {
+  make build-python-apiclient
+}
+
+publish_python_apiclient() {
+  make release-python-apiclient
+}
+
+
+main () {
+  setup_environment_variables
+  download_swagger
+  build_python_apiclient
+
+  if [ -n "$BUILDKITE_TAG" ]; then
+    publish_python_apiclient
+  fi
+}
+
+main

buildkite/scripts/build_python_sdk.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -e
+
+setup_environment_variables() {
+  export PYPI_TOKEN=$(buildkite-agent secret get PYPI_TOKEN)
+  export TEST_PYPI_TOKEN=$(buildkite-agent secret get TEST_PYPI_TOKEN)
+  export RELEASE_PYTHON_PACKAGES=1
+}
+
+build_python_sdk() {
+  make build-python-sdk
+}
+
+publish_python_sdk() {
+  make release-python-sdk
+}
+
+main() {
+  setup_environment_variables
+  build_python_sdk
+
+  if [ -n "$BUILDKITE_TAG" ]; then
+    publish_python_sdk
+  fi
+
+}
+
+main

buildkite/scripts/generate_swagger.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -euo pipefail
+
+# Call generate_swagger.sh
+generate_swagger() {
+    ./scripts/generate_swagger.sh
+}
+
+upload_swagger() {
+    cd docs
+    buildkite-agent artifact upload "swagger.json"
+}
+
+main() {
+    generate_swagger
+    upload_swagger
+}
+
+main

clients/Earthfile

@@ -84,10 +84,9 @@ pypi-upload:
     BUILD +install-twine
 
     ARG RELEASE_PYTHON_PACKAGES
-    LET REPOSITORY_FLAG = "--repository-url https://test.pypi.org/legacy/"
     IF [ "${RELEASE_PYTHON_PACKAGES}" = 1 ]
         RUN echo "Publishing to pypi.org"
-        RUN REPOSITORY_FLAG = "" # Publish to default repository
+        LET REPOSITORY_FLAG = "" # Publish to default repository
         RUN --secret PYPI_TOKEN test -n "${PYPI_TOKEN}" || (echo "PYPI_TOKEN not found" && exit 1)
         RUN --secret PYPI_TOKEN python3 -m twine upload \
             --non-interactive \
@@ -99,6 +98,7 @@ pypi-upload:
             ${REPOSITORY_FLAG}
     ELSE
         RUN echo "Publishing to test.pypi.org"
+        LET REPOSITORY_FLAG = "--repository-url https://test.pypi.org/legacy/"
         RUN --secret TEST_PYPI_TOKEN test -n "${TEST_PYPI_TOKEN}" || (echo "TEST_PYPI_TOKEN not found" && exit 1)
         RUN --secret TEST_PYPI_TOKEN python3 -m twine upload \
             --non-interactive \

clients/Makefile

@@ -16,10 +16,10 @@ release:
 	echo "PYPI_VERSION=$(VERSION)" > .arg
 
 	# Get PYPI_TOKEN from environment or .env file and push to .secret/ file
-	python ../scripts/get_pypi_token.py
+	python3 ../scripts/get_pypi_token.py
 
 	$(CP) ${SWAGGER_JSON} ./swagger.json
-	${MAKE} clean && ${EARTHLY} +pypi-upload --PACKAGE_NAME=${PACKAGE_NAME} --SWAGGER_JSON=${SWAGGER_JSON} --VERSION=${VERSION}
+	${MAKE} clean && ${EARTHLY} +pypi-upload --PACKAGE_NAME=${PACKAGE_NAME} --SWAGGER_JSON=${SWAGGER_JSON} --VERSION=${VERSION} --RELEASE_PYTHON_PACKAGES=${RELEASE_PYTHON_PACKAGES}
 	@echo "Python API client released."
 
 .PHONY: all

python/Earthfile

@@ -61,10 +61,9 @@ publish:
     FROM +install-twine
     BUILD +install-twine
     ARG RELEASE_PYTHON_PACKAGES
-    LET REPOSITORY_FLAG = "--repository-url https://test.pypi.org/legacy/"
     IF [ "${RELEASE_PYTHON_PACKAGES}" = 1 ]
         RUN echo "Publishing to pypi.org"
-        RUN REPOSITORY_FLAG = "" # Publish to default repository
+        LET REPOSITORY_FLAG = "" # Publish to default repository
         RUN --secret PYPI_TOKEN test -n "${PYPI_TOKEN}" || (echo "PYPI_TOKEN not found" && exit 1)
         RUN --secret PYPI_TOKEN python3 -m twine upload \
             --non-interactive \
@@ -76,6 +75,7 @@ publish:
             ${REPOSITORY_FLAG}
     ELSE
         RUN echo "Publishing to test.pypi.org"
+        LET REPOSITORY_FLAG = "--repository-url https://test.pypi.org/legacy/"
         RUN --secret TEST_PYPI_TOKEN test -n "${TEST_PYPI_TOKEN}" || (echo "TEST_PYPI_TOKEN not found" && exit 1)
         RUN --secret TEST_PYPI_TOKEN python3 -m twine upload \
             --non-interactive \

python/Makefile

@@ -67,6 +67,6 @@ publish:
 	echo "PYPI_VERSION=$(VERSION)" > .arg
 
 	# Get PYPI_TOKEN from environment or .env file and push to .secret/ file
-	python ../scripts/get_pypi_token.py
+	python3 ../scripts/get_pypi_token.py
 
-	$(EARTHLY) --push +publish --PYPI_VERSION=$(VERSION)
+	$(EARTHLY) --push +publish --RELEASE_PYTHON_PACKAGES=${RELEASE_PYTHON_PACKAGES} --PYPI_VERSION=${PYPI_VERSION}