diff --git a/.circleci/config.yml b/.circleci/config.yml index eb2d0d0a44..f07b2d1f53 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,77 +35,70 @@ commands: parameters: executor: type: enum - enum: - [ - "linux-amd64", - "linux-arm64", - "darwin-amd64", - "darwin-arm64", - "windows-amd64", - ] + enum: ["linux-amd64", "linux-arm64", "darwin-amd64", "darwin-arm64", "windows-amd64"] steps: - - run: - name: Set GOVER - command: | - go_spec=$(grep 'go [[:digit:]].[[:digit:]]*' go.work | cut -d' ' -f2) - version=$(curl --silent --show-error --location --fail "https://go.dev/dl/?mode=json&include=all" | \ - jq --arg v "$go_spec" --raw-output '[.[] | select(.stable) | select(.version | startswith("go"+$v)) | .version | ltrimstr("go")] | sort_by(split(".") | map(tonumber)) | reverse | .[0]') - echo "Latest Go version for $go_spec is $version" - echo "export GOVER=$version" >> $BASH_ENV - - - when: - condition: - equal: ["windows-amd64", << parameters.executor >>] - steps: - - run: - name: Install Go - command: | - rm -rf /c/Program\ Files/Go - curl --silent --show-error --location --fail https://go.dev/dl/go$GOVER.<< parameters.executor >>.zip | tar --extract --gzip --file=- --directory=/c/Program\ Files - - - when: - condition: - or: - - equal: ["darwin-amd64", << parameters.executor >>] - - equal: ["darwin-arm64", << parameters.executor >>] - - equal: ["linux-amd64", << parameters.executor >>] - - equal: ["linux-arm64", << parameters.executor >>] - steps: - - run: - name: Install Go - command: | - sudo rm -fr /usr/local/go /usr/local/bin/go - curl --silent --show-error --location --fail https://go.dev/dl/go$GOVER.<< parameters.executor >>.tar.gz | sudo tar --extract --gzip --file=- --directory=/usr/local - sudo ln -s /usr/local/go/bin/go /usr/local/bin/go - - - run: - name: Set GOCACHE - command: echo "export GOCACHE=$HOME/.cache/go-build" >> $BASH_ENV - - - run: - name: Set GOPATH - command: echo "export GOPATH=$HOME/go" >> $BASH_ENV - - - restore_cache: - name: Restoring Go cache - key: go-mod-{{ arch }}-{{ checksum "go.sum" }} - - - run: - name: Install packages - command: go mod tidy - - - save_cache: - name: Saving Go cache - key: go-mod-{{ arch }}-{{ checksum "go.sum" }} - paths: - - ~/.cache/go-build - - ~/go/pkg/mod + - run: + name: Set GOVER + command: | + go_spec=$(grep 'go [[:digit:]].[[:digit:]]*' go.work | cut -d' ' -f2) + version=$(curl --silent --show-error --location --fail "https://go.dev/dl/?mode=json&include=all" | \ + jq --arg v "$go_spec" --raw-output '[.[] | select(.stable) | select(.version | startswith("go"+$v)) | .version | ltrimstr("go")] | sort_by(split(".") | map(tonumber)) | reverse | .[0]') + echo "Latest Go version for $go_spec is $version" + echo "export GOVER=$version" >> $BASH_ENV + + - when: + condition: + equal: ["windows-amd64", << parameters.executor >>] + steps: + - run: + name: Install Go + command: | + rm -rf /c/Program\ Files/Go + curl --silent --show-error --location --fail https://go.dev/dl/go$GOVER.<< parameters.executor >>.zip | tar --extract --gzip --file=- --directory=/c/Program\ Files + + - when: + condition: + or: + - equal: ["darwin-amd64", << parameters.executor >>] + - equal: ["darwin-arm64", << parameters.executor >>] + - equal: ["linux-amd64", << parameters.executor >>] + - equal: ["linux-arm64", << parameters.executor >>] + steps: + - run: + name: Install Go + command: | + sudo rm -fr /usr/local/go /usr/local/bin/go + curl --silent --show-error --location --fail https://go.dev/dl/go$GOVER.<< parameters.executor >>.tar.gz | sudo tar --extract --gzip --file=- --directory=/usr/local + sudo ln -s /usr/local/go/bin/go /usr/local/bin/go + + - run: + name: Set GOCACHE + command: echo "export GOCACHE=$HOME/.cache/go-build" >> $BASH_ENV + + - run: + name: Set GOPATH + command: echo "export GOPATH=$HOME/go" >> $BASH_ENV + + - restore_cache: + name: Restoring Go cache + key: go-mod-{{ arch }}-{{ checksum "go.sum" }} + + - run: + name: Install packages + command: go mod tidy + + - save_cache: + name: Saving Go cache + key: go-mod-{{ arch }}-{{ checksum "go.sum" }} + paths: + - ~/.cache/go-build + - ~/go/pkg/mod install_python: steps: - - run: sudo apt install python3.10 -y - - run: curl -sSL https://install.python-poetry.org | python3 - - - run: echo 'export PATH="/home/circleci/.local/bin:$PATH"' >> "$BASH_ENV" + - run: sudo apt install python3.10 -y + - run: curl -sSL https://install.python-poetry.org | python3 - + - run: echo 'export PATH="/home/circleci/.local/bin:$PATH"' >> "$BASH_ENV" jobs: test: @@ -121,14 +114,7 @@ jobs: parameters: executor: type: enum - enum: - [ - "linux-amd64", - "linux-arm64", - "darwin-amd64", - "darwin-arm64", - "windows-amd64", - ] + enum: ["linux-amd64", "linux-arm64", "darwin-amd64", "darwin-arm64", "windows-amd64"] target_arch: type: enum enum: ["amd64", "arm64", "armv6", "armv7"] @@ -139,131 +125,131 @@ jobs: type: string default: "" steps: - - checkout - - - attach_workspace: - at: . - - - when: - condition: - equal: ["windows", << parameters.target_os >>] - steps: - - run: - name: Install GNU Make - command: | - choco install -y make - choco install -y jq - shell: powershell.exe - - - install_go: - executor: << parameters.executor >> - - - run: - name: "Setup BACALHAU_ENVIRONMENT environment variable" - command: echo 'export BACALHAU_ENVIRONMENT=test' >> "$BASH_ENV" - - run: - name: Test Go - environment: - LOG_LEVEL: debug - TEST_BUILD_TAGS: << parameters.build_tags >> - TEST_PARALLEL_PACKAGES: 4 # This is set to 4 as xlarge instances have at least 8 CPUs, and we want to leave some CPU for the Docker instances - command: | - export GOBIN=${HOME}/bin - export PATH=$GOBIN:$PATH - go install gotest.tools/gotestsum@v1.8.2 - make test-and-report - no_output_timeout: 20m - - - store_test_results: - path: . - - persist_to_workspace: - root: coverage/ - paths: - - "*.coverage" + - checkout + + - attach_workspace: + at: . + + - when: + condition: + equal: ["windows", << parameters.target_os >>] + steps: + - run: + name: Install GNU Make + command: | + choco install -y make + choco install -y jq + shell: powershell.exe + + - install_go: + executor: << parameters.executor >> + + - run: + name: "Setup BACALHAU_ENVIRONMENT environment variable" + command: echo 'export BACALHAU_ENVIRONMENT=test' >> "$BASH_ENV" + - run: + name: Test Go + environment: + LOG_LEVEL: debug + TEST_BUILD_TAGS: << parameters.build_tags >> + TEST_PARALLEL_PACKAGES: 4 # This is set to 4 as xlarge instances have at least 8 CPUs, and we want to leave some CPU for the Docker instances + command: | + export GOBIN=${HOME}/bin + export PATH=$GOBIN:$PATH + go install gotest.tools/gotestsum@v1.8.2 + make test-and-report + no_output_timeout: 20m + + - store_test_results: + path: . + - persist_to_workspace: + root: coverage/ + paths: + - "*.coverage" test-python-sdk: parallelism: 1 executor: linux-amd64 steps: - - checkout - - install_python - - run: - working_directory: python - command: make setup test + - checkout + - install_python + - run: + working_directory: python + command: make setup test test-python-airflow: parallelism: 1 executor: linux-amd64 steps: - - checkout - - install_python - - run: - working_directory: integration/airflow - command: make setup test-all + - checkout + - install_python + - run: + working_directory: integration/airflow + command: make setup test-all test-python-flyte: parallelism: 1 executor: linux-amd64 steps: - - checkout - - install_python - - run: - working_directory: integration/flyte - command: make setup test + - checkout + - install_python + - run: + working_directory: integration/flyte + command: make setup test build: parallelism: 1 executor: linux-amd64 steps: - - checkout - - - attach_workspace: - at: . - - - install_go: - executor: linux-amd64 - - - run: - name: Build tarball - command: | - echo "$PRIVATE_PEM_B64" | base64 --decode > /tmp/private.pem - echo "$PUBLIC_PEM_B64" | base64 --decode > /tmp/public.pem - export PRIVATE_KEY_PASSPHRASE="$(echo $PRIVATE_KEY_PASSPHRASE_B64 | base64 --decode)" - # Prevent rebuilding web ui, we should have already attached it - find webui -exec touch -c '{}' + - GOOS=linux GOARCH=amd64 make build-bacalhau-tgz - GOOS=linux GOARCH=arm64 make build-bacalhau-tgz - GOOS=darwin GOARCH=amd64 make build-bacalhau-tgz - GOOS=darwin GOARCH=arm64 make build-bacalhau-tgz - GOOS=linux GOARCH=armv6 make build-bacalhau-tgz - GOOS=linux GOARCH=armv7 make build-bacalhau-tgz - GOOS=windows GOARCH=amd64 make build-bacalhau-tgz - - - persist_to_workspace: - root: dist/ - paths: - - "*.tar.gz" - - "*.sha256" - - - store_artifacts: - path: dist/ + - checkout + + - attach_workspace: + at: . + + - install_go: + executor: linux-amd64 + + - run: + name: Build tarball + command: | + echo "$PRIVATE_PEM_B64" | base64 --decode > /tmp/private.pem + echo "$PUBLIC_PEM_B64" | base64 --decode > /tmp/public.pem + export PRIVATE_KEY_PASSPHRASE="$(echo $PRIVATE_KEY_PASSPHRASE_B64 | base64 --decode)" + # Prevent rebuilding web ui, we should have already attached it + find webui -exec touch -c '{}' + + GOOS=linux GOARCH=amd64 make build-bacalhau-tgz + GOOS=linux GOARCH=arm64 make build-bacalhau-tgz + GOOS=darwin GOARCH=amd64 make build-bacalhau-tgz + GOOS=darwin GOARCH=arm64 make build-bacalhau-tgz + GOOS=linux GOARCH=armv6 make build-bacalhau-tgz + GOOS=linux GOARCH=armv7 make build-bacalhau-tgz + GOOS=windows GOARCH=amd64 make build-bacalhau-tgz + + - persist_to_workspace: + root: dist/ + paths: + - "*.tar.gz" + - "*.sha256" + + - store_artifacts: + path: dist/ build_webui: executor: node/default steps: - - checkout - - node/install-packages: - app-dir: webui - - run: - working_directory: webui - environment: - GENERATE_SOURCEMAP: "false" - command: npm run build - - persist_to_workspace: - root: . - paths: - - "webui/build/*" - - "webui/node_modules/*" + - checkout + - node/install-packages: + app-dir: webui + - run: + working_directory: webui + environment: + GENERATE_SOURCEMAP: "false" + command: npm run build + - persist_to_workspace: + root: . + paths: + - "webui/build/*" + - "webui/node_modules/*" build_canary: parallelism: 1 @@ -273,258 +259,209 @@ jobs: working_directory: ~/repo executor: linux-amd64 steps: - - checkout + - checkout - - install_go: - executor: linux-amd64 + - install_go: + executor: linux-amd64 - - run: - name: Install Executor Plugins - command: make install-plugins + - run: + name: Install Executor Plugins + command: make install-plugins - - run: - name: Set canary dependency - working_directory: ops/aws/canary/lambda - command: make update + - run: + name: Set canary dependency + working_directory: ops/aws/canary/lambda + command: make update - - run: - name: Build - working_directory: ops/aws/canary/lambda - command: make build -j + - run: + name: Build + working_directory: ops/aws/canary/lambda + command: make build -j - - run: - name: Run tests - working_directory: ops/aws/canary/lambda - command: | - export GOBIN=${HOME}/bin - export PATH=$GOBIN:$PATH - go install gotest.tools/gotestsum@v1.8.2 - make test + - run: + name: Run tests + working_directory: ops/aws/canary/lambda + command: | + export GOBIN=${HOME}/bin + export PATH=$GOBIN:$PATH + go install gotest.tools/gotestsum@v1.8.2 + make test - - store_test_results: - path: ops/aws/canary/lambda/tests.xml + - store_test_results: + path: ops/aws/canary/lambda/tests.xml coverage: executor: linux-amd64 environment: GOPROXY: https://proxy.golang.org steps: - - checkout + - checkout - - attach_workspace: - at: coverage/ + - attach_workspace: + at: coverage/ - - run: - name: Install gocovmerge - command: go install github.com/wadey/gocovmerge@latest + - run: + name: Install gocovmerge + command: go install github.com/wadey/gocovmerge@latest - - run: - name: Build coverage report - command: make coverage-report + - run: + name: Build coverage report + command: make coverage-report - - store_artifacts: - path: coverage/coverage.html - - lint: - parallelism: 1 - environment: - GOLANGCILINT: v1.51.2 - GOPROXY: https://proxy.golang.org - working_directory: ~/repo - executor: linux-amd64 - steps: - - checkout - - - run: - name: Init tools - command: | - make init - - - install_go: - executor: linux-amd64 - - - run: - name: Install golangci-lint - command: | - echo "Installing GOLANGCILINT: ${GOLANGCILINT}" - # binary will be /usr/local/go/bin/bin/golangci-lint - # For some reason, .circlerc (I don't know where this file is generated) reports `go env GOPATH` as '/home/circleci/.go_workspace:/usr/local/go_workspace' (with the colon) - # This breaks normal pathing. So just installing in ./bin/ - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | BINDIR=${HOME}/bin sh -s ${GOLANGCILINT} - golangci-lint version - - - run: - name: Install Pre-commit - command: | - make install-pre-commit - - - run: - name: Cache Precommit - command: | - cp .pre-commit-config.yaml pre-commit-cache-key.txt - poetry run python --version --version >> pre-commit-cache-key.txt - - - restore_cache: - name: Restoring pre-commit cache - key: v1-pc-cache-{{ checksum "pre-commit-cache-key.txt" }} - - - run: - name: Run linter - command: | - make precommit + - store_artifacts: + path: coverage/coverage.html release: executor: linux-amd64 steps: - - checkout - - attach_workspace: - at: dist/ - - run: - name: Install gh - command: | - wget https://github.com/cli/cli/releases/download/v2.14.7/gh_2.14.7_linux_amd64.deb - sudo dpkg -i gh_2.14.7_linux_amd64.deb - echo "$BACALHAU_RELEASE_TOKEN" | gh auth login --with-token - - run: - name: Uploading to Release - << pipeline.git.tag >> - command: | - TAG="<< pipeline.git.tag >>" - echo "TAG = ${TAG}" - find dist/ - gh release upload $TAG dist/bacalhau_$TAG_* + - checkout + - attach_workspace: + at: dist/ + - run: + name: Install gh + command: | + wget https://github.com/cli/cli/releases/download/v2.14.7/gh_2.14.7_linux_amd64.deb + sudo dpkg -i gh_2.14.7_linux_amd64.deb + echo "$BACALHAU_RELEASE_TOKEN" | gh auth login --with-token + - run: + name: Uploading to Release - << pipeline.git.tag >> + command: | + TAG="<< pipeline.git.tag >>" + echo "TAG = ${TAG}" + find dist/ + gh release upload $TAG dist/bacalhau_$TAG_* release_python: executor: linux-amd64 working_directory: ~/repo steps: - - checkout - - attach_workspace: - at: /tmp/py_dist - - run: - name: Copy dist files - command: | - mkdir -p clients/python/dist - cp /tmp/py_dist/bacalhau_apiclient* clients/python/dist - mkdir -p python/dist - cp /tmp/py_dist/bacalhau_sdk* python/dist - mkdir -p integration/airflow/dist - cp /tmp/py_dist/bacalhau_airflow* integration/airflow/dist - - run: - name: Release python apiclient - command: | - make release-python-apiclient - - run: - name: Install Python SDK pre-requisites - command: | - curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.4.0 python3 - - echo 'export PATH="/home/circleci/.local/bin:$PATH"' >> "$BASH_ENV" - - run: - name: Release python sdk - command: | - make release-python-sdk - - run: - name: Release python Airflow integration - command: | - pip3 install twine==4.0.2 - make release-bacalhau-airflow - - run: - name: Release python Flyte integration - command: | - pip3 install twine==4.0.2 - make release-bacalhau-flyte + - checkout + - attach_workspace: + at: /tmp/py_dist + - run: + name: Copy dist files + command: | + mkdir -p clients/python/dist + cp /tmp/py_dist/bacalhau_apiclient* clients/python/dist + mkdir -p python/dist + cp /tmp/py_dist/bacalhau_sdk* python/dist + mkdir -p integration/airflow/dist + cp /tmp/py_dist/bacalhau_airflow* integration/airflow/dist + - run: + name: Release python apiclient + command: | + make release-python-apiclient + - run: + name: Install Python SDK pre-requisites + command: | + curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.4.0 python3 - + echo 'export PATH="/home/circleci/.local/bin:$PATH"' >> "$BASH_ENV" + - run: + name: Release python sdk + command: | + make release-python-sdk + - run: + name: Release python Airflow integration + command: | + pip3 install twine==4.0.2 + make release-bacalhau-airflow + - run: + name: Release python Flyte integration + command: | + pip3 install twine==4.0.2 + make release-bacalhau-flyte build_swagger: executor: linux-amd64 environment: GOPROXY: https://proxy.golang.org - GOLANGCILINT: v1.51.2 TARGET_COMMIT: << pipeline.git.revision >> working_directory: ~/repo steps: - - checkout - - add_ssh_keys: - fingerprints: - - "b2:46:a2:7c:94:1f:84:be:99:70:1e:44:50:1e:33:2b" - - install_go: - executor: linux-amd64 - - run: - name: Build Swagger - command: | - export GOBIN=${HOME}/bin - export PATH=$GOBIN:$PATH - go install golang.org/x/tools/cmd/stringer - go install go.uber.org/mock/gomock - go install github.com/golang/mock/mockgen - go install github.com/swaggo/swag/cmd/swag@latest - make generate - - - persist_to_workspace: - root: . - paths: - - pkg/swagger/docs.go - - webui/public/swagger/swagger.json - - - when: # Only if the current branch is main, we push the swagger spec to the repo (via auto PR) - condition: - equal: ["main", << pipeline.git.branch >>] - steps: - - run: - name: Install gh - command: | - wget https://github.com/cli/cli/releases/download/v2.14.7/gh_2.14.7_linux_amd64.deb - sudo dpkg -i gh_2.14.7_linux_amd64.deb - echo "$GITHUB_PR_PAT" | gh auth login --with-token - rm gh_2.14.7_linux_amd64.deb - - run: - name: Install human-id generator - command: | - pip3 install human-id - - run: - name: Git commit - command: | - git config user.email "$(git log --format='format:%ae' -n1)" - git config user.name "$(git log --format='format:%an' -n1)" - # Note: we delete this branch after the PR is merged - GH_BRANCH_NAME=(ci-build-swagger-docs_$(humanid-gen --words 3 | grep -oE "^([^-]*-){1}[^-]*")-$(((RANDOM % $((100 - 1))) + 1))) - git checkout -b $GH_BRANCH_NAME - if test -n "$(git ls-files --modified | grep -e '^docs/')"; then - git add --verbose -- ./docs - COMMIT_MSG="Build swagger reference - this is an automatic commit" - git commit -m "[circleci] $COMMIT_MSG [skip ci]" - git push --set-upstream origin $GH_BRANCH_NAME - # Note: if you close the PR below manually, you should delete the `ci-build-swagger-docs_*` branch as well - PR_URL=$(gh pr create --fill --head $(git rev-parse --abbrev-ref HEAD) --base main --label schema --repo bacalhau-project/bacalhau) - echo "Pull request: $PR_URL" - sleep 3 - gh pr merge --auto --delete-branch -r $PR_URL - fi + - checkout + - add_ssh_keys: + fingerprints: + - "b2:46:a2:7c:94:1f:84:be:99:70:1e:44:50:1e:33:2b" + - install_go: + executor: linux-amd64 + - run: + name: Build Swagger + command: | + export GOBIN=${HOME}/bin + export PATH=$GOBIN:$PATH + go install golang.org/x/tools/cmd/stringer + go install go.uber.org/mock/gomock + go install github.com/golang/mock/mockgen + go install github.com/swaggo/swag/cmd/swag@latest + make generate + + - persist_to_workspace: + root: . + paths: + - pkg/swagger/docs.go + - webui/public/swagger/swagger.json + + - when: # Only if the current branch is main, we push the swagger spec to the repo (via auto PR) + condition: + equal: ["main", << pipeline.git.branch >>] + steps: + - run: + name: Install gh + command: | + wget https://github.com/cli/cli/releases/download/v2.14.7/gh_2.14.7_linux_amd64.deb + sudo dpkg -i gh_2.14.7_linux_amd64.deb + echo "$GITHUB_PR_PAT" | gh auth login --with-token + rm gh_2.14.7_linux_amd64.deb + - run: + name: Install human-id generator + command: | + pip3 install human-id + - run: + name: Git commit + command: | + git config user.email "$(git log --format='format:%ae' -n1)" + git config user.name "$(git log --format='format:%an' -n1)" + # Note: we delete this branch after the PR is merged + GH_BRANCH_NAME=(ci-build-swagger-docs_$(humanid-gen --words 3 | grep -oE "^([^-]*-){1}[^-]*")-$(((RANDOM % $((100 - 1))) + 1))) + git checkout -b $GH_BRANCH_NAME + if test -n "$(git ls-files --modified | grep -e '^docs/')"; then + git add --verbose -- ./docs + COMMIT_MSG="Build swagger reference - this is an automatic commit" + git commit -m "[circleci] $COMMIT_MSG [skip ci]" + git push --set-upstream origin $GH_BRANCH_NAME + # Note: if you close the PR below manually, you should delete the `ci-build-swagger-docs_*` branch as well + PR_URL=$(gh pr create --fill --head $(git rev-parse --abbrev-ref HEAD) --base main --label schema --repo bacalhau-project/bacalhau) + echo "Pull request: $PR_URL" + sleep 3 + gh pr merge --auto --delete-branch -r $PR_URL + fi docker: executor: linux-amd64 steps: - - checkout - - attach_workspace: - at: dist/ - - run: - command: | - mkdir -p bin/linux/amd64 && tar -xvf dist/bacalhau_*_linux_amd64.tar.gz -C bin/linux/amd64 - mkdir -p bin/linux/arm64 && tar -xvf dist/bacalhau_*_linux_arm64.tar.gz -C bin/linux/arm64 - - run: - command: | - docker context create buildx-build - docker buildx create --use buildx-build - make build-bacalhau-image - - when: - condition: - matches: - value: << pipeline.git.tag >> - pattern: "^v([0-9]+).([0-9]+).([0-9]+).*$" - steps: - - run: - name: Login to Github Container Registry - command: echo $GHCR_PAT | docker login ghcr.io -u circleci --password-stdin - - run: make push-bacalhau-image + - checkout + - attach_workspace: + at: dist/ + - run: + command: | + mkdir -p bin/linux/amd64 && tar -xvf dist/bacalhau_*_linux_amd64.tar.gz -C bin/linux/amd64 + mkdir -p bin/linux/arm64 && tar -xvf dist/bacalhau_*_linux_arm64.tar.gz -C bin/linux/arm64 + - run: + command: | + docker context create buildx-build + docker buildx create --use buildx-build + make build-bacalhau-image + - when: + condition: + matches: + value: << pipeline.git.tag >> + pattern: "^v([0-9]+).([0-9]+).([0-9]+).*$" + steps: + - run: + name: Login to Github Container Registry + command: echo $GHCR_PAT | docker login ghcr.io -u circleci --password-stdin + - run: make push-bacalhau-image build_python_packages: executor: linux-amd64 @@ -533,153 +470,145 @@ jobs: # PYPI_VERSION: 0.3.24.dev8 # use this to set a custom version identifier (https://peps.python.org/pep-0440/) working_directory: ~/repo steps: - - checkout - - attach_workspace: - at: /tmp/swagger_spec/ - - run: - name: Copy swagger.json from workspace - command: | - cp /tmp/swagger_spec/webui/public/swagger/swagger.json ./docs/swagger.json - - run: - name: Install Python API client pre-requisites - command: | - CODEGEN_BASE_URL="https://repo1.maven.org/maven2/io/swagger/codegen/v3" - wget ${CODEGEN_BASE_URL}/swagger-codegen-cli/3.0.41/swagger-codegen-cli-3.0.41.jar -O ${HOME}/bin/swagger-codegen-cli.jar - chmod +x ${HOME}/bin/swagger-codegen-cli.jar - ${HOME}/bin/swagger-codegen-cli.jar version - - run: - name: Build Python API client - command: | - make build-python-apiclient - - persist_to_workspace: - root: clients/python/dist - paths: - - bacalhau_apiclient-*.tar.gz - - bacalhau_apiclient-*.whl - - run: - name: Install Python SDK pre-requisites - command: | - curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.4.0 python3 - - echo 'export PATH="/home/circleci/.local/bin:$PATH"' >> "$BASH_ENV" - cd python - /home/circleci/.local/bin/poetry run pip3 install black>=22.12.0 isort>=5.8.0 flake8>=6.0.0 mypy>=0.991 pre-commit>=2.12.0 - - run: - name: Build Python SDK - command: | - make build-python-sdk - - persist_to_workspace: - root: python/dist - paths: - - bacalhau_sdk-*.tar.gz - - bacalhau_sdk-*.whl - - run: - name: Install bacalhau-airflow pre-requisites - command: | - cd integration/airflow - pip3 install -r dev-requirements.txt - - run: - name: Build Python Bacalhau Airflow - command: | - make build-bacalhau-airflow - - run: - name: Install Flyte plugin pre-requisites - command: | - cd integration/flyte - # installs flytekit, flyteidl and deps. - make setup - - run: - name: Build Python Bacalhau Flyte plugin - command: | - cd integration/flyte/plugins/ - make build_all_plugins - - persist_to_workspace: - root: integration/airflow/dist - paths: - - bacalhau_airflow-*.tar.gz - - bacalhau_airflow-*.whl + - checkout + - attach_workspace: + at: /tmp/swagger_spec/ + - run: + name: Copy swagger.json from workspace + command: | + cp /tmp/swagger_spec/webui/public/swagger/swagger.json ./docs/swagger.json + - run: + name: Install Python API client pre-requisites + command: | + CODEGEN_BASE_URL="https://repo1.maven.org/maven2/io/swagger/codegen/v3" + wget ${CODEGEN_BASE_URL}/swagger-codegen-cli/3.0.41/swagger-codegen-cli-3.0.41.jar -O ${HOME}/bin/swagger-codegen-cli.jar + chmod +x ${HOME}/bin/swagger-codegen-cli.jar + ${HOME}/bin/swagger-codegen-cli.jar version + - run: + name: Build Python API client + command: | + make build-python-apiclient + - persist_to_workspace: + root: clients/python/dist + paths: + - bacalhau_apiclient-*.tar.gz + - bacalhau_apiclient-*.whl + - run: + name: Install Python SDK pre-requisites + command: | + curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.4.0 python3 - + echo 'export PATH="/home/circleci/.local/bin:$PATH"' >> "$BASH_ENV" + cd python + /home/circleci/.local/bin/poetry run pip3 install black>=22.12.0 isort>=5.8.0 flake8>=6.0.0 mypy>=0.991 pre-commit>=2.12.0 + - run: + name: Build Python SDK + command: | + make build-python-sdk + - persist_to_workspace: + root: python/dist + paths: + - bacalhau_sdk-*.tar.gz + - bacalhau_sdk-*.whl + - run: + name: Install bacalhau-airflow pre-requisites + command: | + cd integration/airflow + pip3 install -r dev-requirements.txt + - run: + name: Build Python Bacalhau Airflow + command: | + make build-bacalhau-airflow + - run: + name: Install Flyte plugin pre-requisites + command: | + cd integration/flyte + # installs flytekit, flyteidl and deps. + make setup + - run: + name: Build Python Bacalhau Flyte plugin + command: | + cd integration/flyte/plugins/ + make build_all_plugins + - persist_to_workspace: + root: integration/airflow/dist + paths: + - bacalhau_airflow-*.tar.gz + - bacalhau_airflow-*.whl # Invoke jobs via workflows # See: https://circleci.com/docs/2.0/configuration-reference/#workflows workflows: - lint: - jobs: - - lint: - name: Run linters and static checkers - filters: - tags: - ignore: /.*/ - check_canary: jobs: - - build_canary: - name: Check canary build - filters: - tags: - ignore: /.*/ + - build_canary: + name: Check canary build + filters: + tags: + ignore: /.*/ test: jobs: - - build_webui - - test: - name: test-<< matrix.target_os >>-<< matrix.target_arch >>-<< matrix.build_tags >> - executor: << matrix.target_os >>-<< matrix.target_arch >> - requires: - - build_webui - matrix: - parameters: - target_os: ["linux", "darwin", "windows"] - target_arch: ["amd64", "arm64"] - build_tags: ["unit", "integration"] - exclude: - - target_os: windows - target_arch: arm64 - build_tags: unit - - target_os: windows - target_arch: arm64 - build_tags: integration - - test-python-sdk - - test-python-airflow - - test-python-flyte - - coverage: - name: Build coverage report - requires: - - test + - build_webui + - test: + name: test-<< matrix.target_os >>-<< matrix.target_arch >>-<< matrix.build_tags >> + executor: << matrix.target_os >>-<< matrix.target_arch >> + requires: + - build_webui + matrix: + parameters: + target_os: ["linux", "darwin", "windows"] + target_arch: ["amd64", "arm64"] + build_tags: ["unit", "integration"] + exclude: + - target_os: windows + target_arch: arm64 + build_tags: unit + - target_os: windows + target_arch: arm64 + build_tags: integration + - test-python-sdk + - test-python-airflow + - test-python-flyte + - coverage: + name: Build coverage report + requires: + - test python: jobs: - - build_swagger: - name: build-swagger-spec - filters: &build_on_branch_and_tag_push - tags: - # only run on tags that look like release tags e.g. v0.1.2 or - # v0.1.3alpha19 (actually v0.1.3anything...) - only: /^v([0-9]+).([0-9]+).([0-9]+).*$/ - - build_python_packages: - name: build-python-packages - requires: [build-swagger-spec] - filters: *build_on_branch_and_tag_push - - release_python: - name: release-python-packages - requires: - - build-swagger-spec - - build-python-packages - # This job will only run on tags (specifically starting with 'v.') and - # will not run on branches - filters: &build_on_release_only - branches: - ignore: /.*/ # don't run on any branches - only tags - <<: *build_on_branch_and_tag_push + - build_swagger: + name: build-swagger-spec + filters: &build_on_branch_and_tag_push + tags: + # only run on tags that look like release tags e.g. v0.1.2 or + # v0.1.3alpha19 (actually v0.1.3anything...) + only: /^v([0-9]+).([0-9]+).([0-9]+).*$/ + - build_python_packages: + name: build-python-packages + requires: [build-swagger-spec] + filters: *build_on_branch_and_tag_push + - release_python: + name: release-python-packages + requires: + - build-swagger-spec + - build-python-packages + # This job will only run on tags (specifically starting with 'v.') and + # will not run on branches + filters: &build_on_release_only + branches: + ignore: /.*/ # don't run on any branches - only tags + !!merge <<: *build_on_branch_and_tag_push build: jobs: - - build_webui: - filters: *build_on_branch_and_tag_push - - build: - requires: [build_webui] - filters: *build_on_branch_and_tag_push - - docker: - requires: [build] - filters: *build_on_branch_and_tag_push - - release: - requires: [build] - filters: *build_on_release_only \ No newline at end of file + - build_webui: + filters: *build_on_branch_and_tag_push + - build: + requires: [build_webui] + filters: *build_on_branch_and_tag_push + - docker: + requires: [build] + filters: *build_on_branch_and_tag_push + - release: + requires: [build] + filters: *build_on_release_only diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000000..131614d440 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,31 @@ +name: Lints + +on: + pull_request: + push: + branches: [main] + +jobs: + golint: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + # Setup nodejs and build the webui so that go vet can + # compile the go code that embeds the webui. + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: 21 + - run: cd webui && npm install + - run: cd webui && npm run build + + - uses: actions/setup-go@v4 + with: + go-version: '1.20.6' + cache: false + - run: go mod tidy + - run: go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55.2 + - uses: pre-commit/action@v3.0.0 +