diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000..b4532b2d82 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,77 @@ +name: Build Bacalhau Binaries + +on: + workflow_call: + +jobs: + build: + name: Build Binary + runs-on: ubuntu-latest + outputs: + status: ${{ job.status }} + strategy: + matrix: + include: + - os: linux + goarch: amd64 + - os: linux + goarch: arm64 + - os: linux + goarch: armv7 + - os: linux + goarch: armv6 + - os: darwin + goarch: amd64 + - os: darwin + goarch: arm64 + - os: windows + goarch: amd64 + + steps: + - name: Install earthly + uses: earthly/actions-setup@v1 + + - uses: actions/checkout@v4 + with: + fetch-depth: "0" # Need to fetch all due to how bacalhau constructs semver + + - uses: actions/setup-go@v5 + with: + go-version-file: go.work + + - name: Build + env: + GOOS: ${{ matrix.os }} + GOARCH: ${{ matrix.goarch }} + PRIVATE_PEM: ${{ secrets.PRIVATE_PEM }} + PUBLIC_PEM: ${{ secrets.PUBLIC_PEM }} + PRIVATE_KEY_PASSPHRASE: ${{ secrets.PRIVATE_KEY_PASSPHRASE }} + run: | + # Add Keys to expected files + echo "${PRIVATE_PEM}" > /tmp/private.pem && chmod 600 /tmp/private.pem + echo "${PUBLIC_PEM}" > /tmp/public.pem && chmod 600 /tmp/public.pem + + # Start build + echo "==> Building bacalhau binary for: ${GOOS} ${GOARCH}..." + make build-bacalhau-tgz + echo "===> Done building bacalhau binary." + + # Listing Builds + echo "===> Built Artifacts:" + ls -lh dist/ + + # Remove keys, good security practice + rm /tmp/private.pem /tmp/public.pem + + - name: Upload binary artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.os }}-${{ matrix.goarch }} + path: "dist/bacalhau_*" + retention-days: 1 # Short retention since these are intermediate artifacts, also save money + + - name: Report build status + if: always() + run: | + echo "Build completed for ${{ matrix.os }}-${{ matrix.goarch }}" + echo "Status: ${{ job.status }}" \ No newline at end of file diff --git a/.github/workflows/cspell.yml b/.github/workflows/cspell.yml index 0a88b458fc..3358731650 100644 --- a/.github/workflows/cspell.yml +++ b/.github/workflows/cspell.yml @@ -2,12 +2,7 @@ name: Codespell on: - push: - branches: [main] - pull_request: - branches: [main] - repository_dispatch: - types: [ok-to-test] + workflow_call: jobs: codespell: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4b6d22816a..b6ba969abf 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,13 +1,32 @@ name: Lints on: - pull_request: - push: - branches: [main] - repository_dispatch: - types: [ok-to-test] + workflow_call: jobs: + # A separate Go-Lint job since the current lint job is heavily + # coupled with a lot of tooling, which will be removed and + # simplified, one step at a time. + # For now, we are just extracting the go-lint. + go-lint: + strategy: + matrix: + os: [ ubuntu-latest ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.work + - name: Setup Dummy WebUi Build + if: matrix.os == 'ubuntu-latest' + run: | + mkdir -p webui/build && touch webui/build/index.html + - uses: golangci/golangci-lint-action@v6 + with: + skip-cache: true + + # TODO: Decouple Each Component Linting lint: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/pipeline-main.yml b/.github/workflows/pipeline-main.yml new file mode 100644 index 0000000000..8ebc6c3674 --- /dev/null +++ b/.github/workflows/pipeline-main.yml @@ -0,0 +1,31 @@ +name: Main Pipeline + +on: + push: + branches: ["eng-292/jamlo/improve-go-linting-in-ci"] +# pull_request: +# branches: [main] +# repository_dispatch: +# types: [ok-to-test] + +jobs: + build: + uses: ./.github/workflows/build.yml + secrets: inherit + + lint: + uses: ./.github/workflows/lint.yml + + swagger-validation: + uses: ./.github/workflows/swagger-validation.yml + + cspell: + uses: ./.github/workflows/cspell.yml + + testcontainers-suite: + needs: [build, lint, swagger-validation, cspell] + if: success() + uses: ./.github/workflows/testcontainers-suite.yml + with: + os: linux + arch: amd64 diff --git a/.github/workflows/swagger-validation.yml b/.github/workflows/swagger-validation.yml index 8d70e7ea3b..92e12f31ce 100644 --- a/.github/workflows/swagger-validation.yml +++ b/.github/workflows/swagger-validation.yml @@ -1,9 +1,7 @@ name: Swagger Validation on: - pull_request: - branches: - - main + workflow_call: jobs: ensure-up-to-date: @@ -17,35 +15,20 @@ jobs: with: fetch-depth: 1 - # Detect Go version from go.mod - - name: Detect Go version from go.mod - id: detect-go-version - shell: bash - run: | - set -euo pipefail - go_version=$(grep '^go ' go.mod | awk '{print $2}') - echo "Go version detected: $go_version" - echo "golang-version=$go_version" >> $GITHUB_ENV - - # Setup Go using the detected version - name: Setup Go uses: actions/setup-go@v5 with: - go-version: ${{ env.golang-version }} + go-version-file: go.work cache: true # Install swag (Swagger generator) - name: Install Swag - shell: bash run: | - set -euo pipefail go install github.com/swaggo/swag/cmd/swag@latest # Verify swag installation - name: Verify Swag Installation - shell: bash run: | - set -euo pipefail if ! command -v swag &> /dev/null; then echo "Swag is not installed. Please ensure Go is properly configured and Swag is installed." exit 1 @@ -53,14 +36,10 @@ jobs: # Generate the swagger.json - name: Generate Swagger file - shell: bash run: | - set -euo pipefail make generate-swagger # Compare the newly generated swagger.json with the committed swagger.json - name: Check for Swagger differences - shell: bash run: | - set -euo pipefail git diff --exit-code pkg/swagger/swagger.json || (echo "Swagger is outdated. Please regenerate it with 'make generate-swagger' and commit the changes." && exit 1) diff --git a/.github/workflows/testcontainers-integration-tests.yml b/.github/workflows/testcontainers-integration-tests.yml deleted file mode 100644 index 5a28cdff1a..0000000000 --- a/.github/workflows/testcontainers-integration-tests.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Test Containers Integration Tests - -on: - push: - branches: [main] - pull_request: - branches: [main] - repository_dispatch: - types: [ok-to-test] - -jobs: - test-build-and-run-testcontainers: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Install Golang - uses: actions/setup-go@v5 - with: - go-version-file: go.work - - - name: Run Go tests in integration test directory - shell: bash - run: | - set -euo pipefail - # TODO: make WebUI happy when compiling - mkdir -p webui/build - echo "hello test" > webui/build/index.html - cd test_integration - go test -v -count=1 ./... diff --git a/.github/workflows/testcontainers-suite.yml b/.github/workflows/testcontainers-suite.yml new file mode 100644 index 0000000000..7e1a621d5a --- /dev/null +++ b/.github/workflows/testcontainers-suite.yml @@ -0,0 +1,41 @@ +name: TestContainers Suite + +on: + workflow_call: + inputs: + os: + required: true + type: string + arch: + required: true + type: string + +jobs: + tests: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Download compiled binary + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.os }}-${{ inputs.arch }} + path: compiled-artifacts/ + + - name: Install Golang + uses: actions/setup-go@v5 + with: + go-version-file: go.work + + - name: Run Go tests in integration test directory + shell: bash + run: | + echo "===> Copy compiled artifact to use in docker images..." + cd compiled-artifacts + tar -xzf bacalhau_*.tar.gz + cp ./bacalhau ../test_integration/common_assets/bacalhau_bin + + echo "===> Running tests..." + cd ../test_integration + go test -v -count=1 ./... diff --git a/.github/workflows/trigger-development-deployment.yaml b/.github/workflows/trigger-development-deployment.yaml deleted file mode 100644 index 848e249efe..0000000000 --- a/.github/workflows/trigger-development-deployment.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: Trigger Deployment to Development on Merge to Main - -on: - push: - branches: - - main - -jobs: - trigger-deployment: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Trigger Deployment - shell: bash - run: | - set -euo pipefail - curl -X POST \ - -H "Authorization: token ${{ secrets.BACALHAU_VMI_DEPLOYMENT_SECRET }}" \ - -H "Accept: application/vnd.github+json" \ - "https://api.github.com/repos/bacalhau-project/bacalhau-vmi/actions/workflows/deploy-commit.yaml/dispatches" \ - --data '{"ref": "main", "inputs": {"commit_reference": "${{ github.sha }}", "redeploy": "false"}}' diff --git a/.gitprecommit/golangci-lint.sh b/.gitprecommit/golangci-lint.sh deleted file mode 100755 index 419f0e3dfa..0000000000 --- a/.gitprecommit/golangci-lint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -golangci-lint run --config=.golangci.yml --timeout 10m diff --git a/.golangci.yml b/.golangci.yml index 641b6e1d6e..530754c783 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -69,7 +69,7 @@ linters: - depguard - dogsled - errcheck - - exportloopref + - copyloopvar - funlen - gochecknoinits - goconst @@ -100,7 +100,7 @@ run: # concurrency: 16 # Timeout for analysis, e.g. 30s, 5m. # Default: 1m - timeout: 5m + timeout: 10m # Exit code when at least one issue was found. # Default: 1 issues-exit-code: 2 diff --git a/buildkite/scripts/lint.sh b/buildkite/scripts/lint.sh index 67231543fd..11c40c420c 100644 --- a/buildkite/scripts/lint.sh +++ b/buildkite/scripts/lint.sh @@ -6,9 +6,6 @@ set -x # NB(forrest/udit): this step needs to be done before linting as without it the code doesn't compile since webuid/build DNE. make build-webui -# NB(forrest/udit): linting cannot be done by pre-commit because it doesn't work... -make lint - # TODO(forrest/udit): deprecate pre-commit and replace it with the individual steps required to validate the code. # e.g. modtidy check, credentials check, go fmt, test file header validation. pre-commit run --show-diff-on-failure --color=always --all-files diff --git a/cmd/util/flags/types.go b/cmd/util/flags/types.go index 8984b86b66..7f64ac5b4f 100644 --- a/cmd/util/flags/types.go +++ b/cmd/util/flags/types.go @@ -96,7 +96,6 @@ func (s *ArrayValueFlag[T]) Set(input string) error { func (s *ArrayValueFlag[T]) String() string { strs := make([]string, 0, len(*s.value)) for _, spec := range *s.value { - spec := spec strs = append(strs, s.stringer(&spec)) } return strings.Join(strs, ", ") @@ -152,7 +151,6 @@ func (s *MapValueFlag[K, V]) Set(input string) error { func (s *MapValueFlag[K, V]) String() string { strs := make([]string, len(*s.value)) for key, value := range *s.value { - key, value := key, value strs = append(strs, s.stringer(&key, &value)) } return strings.Join(strs, ", ") diff --git a/pkg/executor/results.go b/pkg/executor/results.go index 981407c503..53c2433c3d 100644 --- a/pkg/executor/results.go +++ b/pkg/executor/results.go @@ -139,7 +139,6 @@ func WriteJobResults( wg := multierrgroup.Group{} for _, output := range outputs { - output := output wg.Go(func() error { return writeOutputResult(resultsDir, output) }) diff --git a/pkg/test/scenario/results.go b/pkg/test/scenario/results.go index 8f466cde39..ea69d9c12d 100644 --- a/pkg/test/scenario/results.go +++ b/pkg/test/scenario/results.go @@ -132,7 +132,6 @@ func ManyChecks(checks ...CheckResults) CheckResults { return func(resultsDir string) error { var wg multierrgroup.Group for _, check := range checks { - check := check wg.Go(func() error { return check(resultsDir) }) } return wg.Wait() diff --git a/test_integration/README.md b/test_integration/README.md index 35bcc5c076..19779430e1 100644 --- a/test_integration/README.md +++ b/test_integration/README.md @@ -38,5 +38,5 @@ Then: ```shell cd test_integration -go test -v -count=1 ./... +./run_locally.sh ``` diff --git a/test_integration/init_test.go b/test_integration/init_test.go index 22f6d92840..2364b27df9 100644 --- a/test_integration/init_test.go +++ b/test_integration/init_test.go @@ -2,7 +2,6 @@ package test_integration import ( "bacalhau/integration_tests/utils" - "context" "github.com/google/uuid" "log" "os" @@ -22,15 +21,6 @@ func TestMain(m *testing.M) { os.Exit(1) } - ctx := context.Background() - err = utils.CompileBacalhau(ctx, "../main.go") - if err != nil { - log.Println("Error compiling the bacalhau binary: ", err.Error()) - os.Exit(1) - } - - // TODO: Maybe we do not need to created images, but just inject - // TODO: them with artifacts before container starts the starts (certs and binary and configs) err = utils.BuildBaseImages(globalTestExecutionId) if err != nil { log.Println("Error building base images: ", err.Error()) @@ -45,8 +35,6 @@ func TestMain(m *testing.M) { os.Exit(1) } - // TODO: Better cleaning - os.Remove("./common_assets/bacalhau_bin") //Exit with the same code as the test run os.Exit(exitCode) } diff --git a/test_integration/run_locally.sh b/test_integration/run_locally.sh new file mode 100755 index 0000000000..636fe3234f --- /dev/null +++ b/test_integration/run_locally.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -euo pipefail + +# Setup cleanup trap +cleanup() { + echo "Cleaning up..." + rm -f common_assets/bacalhau_bin +} +trap cleanup EXIT + +# cd to main repo +cd .. + +# Get GOARCH value and print it for verification +GOARCH=$(go env GOARCH) +echo "Building for architecture: ${GOARCH}" + +# Remove old binary if exists +rm -f "bin/linux/${GOARCH}/bacalhau" + +# Remove old binary from test_integration directory +rm -f test_integration/common_assets/bacalhau_bin + +# Run make build with GOOS=linux +GOOS=linux make build + +# Verify the binary exists +if [[ ! -f "bin/linux/${GOARCH}/bacalhau" ]]; then + echo "Error: Binary was not created at bin/linux/${GOARCH}/bacalhau" + exit 1 +fi + +# Copy binary to test assets directory +mkdir -p test_integration/common_assets +cp "bin/linux/${GOARCH}/bacalhau" test_integration/common_assets/bacalhau_bin + +cd test_integration || exit + +# Run tests +go test -v -count=1 ./... diff --git a/test_integration/utils/test_utils.go b/test_integration/utils/test_utils.go index dde9b7255e..71ba5fb952 100644 --- a/test_integration/utils/test_utils.go +++ b/test_integration/utils/test_utils.go @@ -11,11 +11,8 @@ import ( dc "github.com/docker/docker/client" "github.com/docker/docker/pkg/archive" "github.com/stretchr/testify/suite" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" "io" "os" - "path/filepath" "regexp" "strings" ) @@ -99,82 +96,6 @@ func DeleteDockerTestImagesAndPrune(suffix string) error { return nil } -func CompileBacalhau(ctx context.Context, programPath string) error { - programDir, err := filepath.Abs(filepath.Dir(programPath)) - if err != nil { - return fmt.Errorf("compiling bacalhau: failed to get absolute path: %w", err) - } - - // Create a container request - // TODO: Improve how we build our binary - req := testcontainers.ContainerRequest{ - Image: "golang:1.23", - Cmd: []string{ - "go", - "build", - "-buildvcs=false", - "-o", "/usr/src/bacalhau-binary", - "-ldflags", "-linkmode external -extldflags '-static'", - "-a", - "./"}, - Files: []testcontainers.ContainerFile{ - { - HostFilePath: programDir, - ContainerFilePath: "/usr/src/", - FileMode: 0755, - }, - }, - WorkingDir: "/usr/src/bacalhau", - WaitingFor: wait.ForExit(), - } - - // Start the container - container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - if err != nil { - return fmt.Errorf("compiling bacalhau: failed to start container: %w", err) - } - defer container.Terminate(ctx) - - // Get the logs - logs, err := container.Logs(ctx) - if err != nil { - return fmt.Errorf("compiling bacalhau: failed to get container logs: %w", err) - } - defer logs.Close() - - // Print the logs - if _, err := io.Copy(os.Stdout, logs); err != nil { - return fmt.Errorf("compiling bacalhau: failed to print logs: %w", err) - } - - // Copy the compiled binary from the container to the host - reader, err := container.CopyFileFromContainer(ctx, "/usr/src/bacalhau-binary") - if err != nil { - return fmt.Errorf("compiling bacalhau: failed to copy file from container: %w", err) - } - defer reader.Close() - - // Create the output file on the host - outFile, err := os.Create(filepath.Join(programDir, "test_integration", "common_assets", "bacalhau_bin")) - if err != nil { - return fmt.Errorf("compiling bacalhau: failed to create output file: %w", err) - } - defer outFile.Close() - - // Copy the content from the reader to the output file - _, err = io.Copy(outFile, reader) - if err != nil { - return fmt.Errorf("compiling bacalhau: failed to write compiled program to file: %w", err) - } - - fmt.Println("Compiled program has been copied to the host machine.") - - return nil -} - func BuildBaseImages(testIdentifier string) error { err := buildDockerImage( "common_assets/dockerfiles/Dockerfile-JumpboxNode",