Support TestContainers PoC (#4627)

**This PR adds a test suite utilizing TestContainers. It support
spinning up docker compose stacks with all bacalhau components, thus
resembling production setup.**

This is not a complete TestSuite, but a big initial step towards
consolidating out testing and evaluating TestContainers usage.

1. Allow us to run integration tests against a docker compose
deployment of bacalhau

2. Bacalhau compute/requestor configuration can be injected in the
container before start. We do not need to rebuild it every time

3. Tests are separated into suites which allows the combination of
similar tests together that will use the same docker compose
deployment. Help speed up things

4. Docker compose deployment has a local registry as well as a minio
server running. This expands out testing ability and scope.

5. Test Suite Works with Github Actions

Linear issue tracker: https://linear.app/expanso/issue/ENG-263/explore-testcontainers-to-utilize-in-testing-strategy
Github Issue: #4597

.cspell/custom-dictionary.txt

@@ -431,4 +431,6 @@ frrist
 swaggo
 isbadactor
 installationid
-firstbacalhauimage
+firstbacalhauimage
+RYUK
+buildvcs

.github/workflows/testcontainers-integration-tests.yml

@@ -0,0 +1,38 @@
+name: Test Containers Integration Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  repository_dispatch:
+    types: [ok-to-test]
+
+jobs:
+  test-build-and-run-testcontainers:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      # Detect Go version from test_integration/go.mod
+      - name: Detect Go version from test_integration/go.mod
+        id: detect-test-go-version
+        run: |
+          go_version=$(grep '^go ' test_integration/go.mod | awk '{print $2}')
+          echo "Go version detected for integration tests: $go_version"
+          echo "GOLANG_TEST_VERSION=$go_version" >> $GITHUB_ENV
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GOLANG_TEST_VERSION }}
+          cache: true
+
+      - name: Run Go tests in integration test directory
+        run: |
+          # TODO: make WebUI happy when compiling
+          mkdir -p webui/build
+          echo "hello test" > webui/build/index.html
+          cd test_integration
+          go test -v -count=1 ./...

.gitignore

@@ -65,3 +65,6 @@ dist/
 .arg
 .ruff_cache
 .aider*
+
+# Integration Tests
+test_integration/assets/dockerfiles/bacalhau_bin

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
   - id: detect-aws-credentials
     args: [--allow-missing-credentials]
   - id: detect-private-key
-    exclude: 'testdata/.*|test-integration/certificates/.*'
+    exclude: 'testdata/.*|test_integration/assets/dockerfiles/certificates/.*|docker-compose-deployment/certificates/.*'
   - id: check-yaml
   - id: check-json
 - repo: https://github.com/astral-sh/ruff-pre-commit
@@ -36,4 +36,5 @@ repos:
   - id: "go-test-build-header"
     name: "Go test '//go:build' header present"
     entry: ".gitprecommit/go_test_build_header.sh"
+    exclude: 'test_integration/.*|'
     language: "script"

go.work

@@ -2,4 +2,5 @@ go 1.23.0
 
 use (
 	.
+	test_integration
 )

test_integration/README.md

@@ -0,0 +1,42 @@
+# Integration Test Suite for Bacalhau
+
+This test suite is designed to perform integration testing for Bacalhau using real docker containers, simulating a production-like environment.
+
+## Key Features
+
+- Uses TestContainers Go library for managing Docker environments
+- Implements Testify Go testing suite for structured and extensible tests
+- Compiles project binary and builds custom Docker images for testing
+- Provides a base suite with common functionality, easily extendable for specific test cases
+- Simulates real production usage by executing commands in a jumpbox container
+
+## Architecture
+
+1. **Base Suite**: Compiles the project binary and builds several Docker images:
+    - Compute node
+    - Requester node
+    - Jumpbox node
+
+2. **TestContainers**: Used to set up and manage a Docker Compose stack for each test suite.
+
+3. **Test Structure**:
+    - Each test suite inherits from the base suite
+    - Suites have their own Docker Compose stack, shared by all tests within the suite
+    - Tests within a suite run in series
+    - Different suites can run in parallel if needed
+
+4. **Test Execution**:
+    - Tests use TestContainers' exec-in-container functionality
+    - Commands are run in the jumpbox container, simulating real-world usage
+    - No mocking is used, providing high-fidelity test results
+
+## Run Test Suite:
+
+To run test suite, you will need to have docker daemon on your local machine, as well as docker compose,
+
+Then:
+
+```shell
+cd test_integration
+go test -v -count=1 ./...
+```

test_integration/assets/docker_compose_files/docker-compose-with-minio-and-registry.yml

@@ -0,0 +1,102 @@
+x-common-env-variables: &common-env-variables
+  NETWORK_AUTH_TOKEN: "i_am_very_secret_token"
+  BACALHAU_API_PORT: "1234"
+  MINIO_ROOT_USER: "minioadmin"
+  MINIO_ROOT_PASSWORD: "minioadminpass"
+  AWS_ACCESS_KEY_ID: "minioadmin"
+  AWS_SECRET_ACCESS_KEY: "minioadminpass"
+
+networks:
+  bacalhau-network:
+    driver: bridge
+
+services:
+  bacalhau-minio-node:
+    image: quay.io/minio/minio
+    command: server /data --console-address ":9001"
+    restart: always
+    networks:
+      - bacalhau-network
+    environment: *common-env-variables
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
+      interval: 1s
+      timeout: 5s
+      retries: 30
+      start_period: 2s
+
+  bacalhau-container-img-registry-node:
+    image: "placeholder-registry-image"
+    restart: always
+    networks:
+      - bacalhau-network
+    environment:
+      REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY: /var/lib/registry
+      REGISTRY_HTTP_ADDR: "0.0.0.0:5000"
+      REGISTRY_HTTP_TLS_CERTIFICATE: "/certs/bacalhau-container-img-registry-node.crt"
+      REGISTRY_HTTP_TLS_KEY: "/certs/bacalhau-container-img-registry-node.key"
+    healthcheck:
+      test: [ "CMD-SHELL", "nc -zv localhost 5000" ]
+      interval: 1s
+      timeout: 5s
+      retries: 30
+      start_period: 2s
+
+  bacalhau-requester-node:
+    image: "placeholder-requester-image"
+    networks:
+      - bacalhau-network
+    environment: *common-env-variables
+    depends_on:
+      bacalhau-minio-node:
+        condition: service_healthy
+    privileged: true
+    command:
+      - /bin/bash
+      - -c
+      - |
+        bacalhau config set "orchestrator.auth.token" "$${NETWORK_AUTH_TOKEN}" && bacalhau serve --orchestrator -c api.port=$${BACALHAU_API_PORT}
+    healthcheck:
+      test: [ "CMD-SHELL", "nc -zv localhost 1234" ]
+      interval: 1s
+      timeout: 5s
+      retries: 30
+      start_period: 2s
+
+  bacalhau-compute-node:
+    image: "placeholder-compute-image"
+    privileged: true
+    networks:
+      - bacalhau-network
+    depends_on:
+      bacalhau-requester-node:
+        condition: service_healthy
+      bacalhau-container-img-registry-node:
+        condition: service_healthy
+    environment:
+      <<: *common-env-variables
+      REQUESTER_NODE_LINK: 'bacalhau-requester-node'
+    healthcheck:
+      test: [ "CMD-SHELL", "nc -zv localhost 1234" ]
+      interval: 1s
+      timeout: 5s
+      retries: 30
+      start_period: 2s
+
+  bacalhau-jumpbox-node:
+    image: "placeholder-jumpbox-image"
+    privileged: true
+    networks:
+      - bacalhau-network
+    depends_on:
+      bacalhau-requester-node:
+        condition: service_healthy
+      bacalhau-compute-node:
+        condition: service_healthy
+      bacalhau-container-img-registry-node:
+        condition: service_healthy
+    environment:
+      <<: *common-env-variables
+      BACALHAU_API_HOST: 'bacalhau-requester-node'
+      BACALHAU_COMPUTE_NODE_HOST: 'bacalhau-compute-node'
+      BACALHAU_MINIO_NODE_HOST: 'bacalhau-minio-node'

test_integration/assets/dockerfiles/Dockerfile-ComputeNode

@@ -0,0 +1,25 @@
+# Use the docker:dind image as the base image
+FROM docker:dind
+
+# Set the working directory
+WORKDIR /app
+
+# Install curl and bash
+RUN apk update && apk add --no-cache curl bash
+
+# Install the ca-certificates package
+RUN apk add --no-cache ca-certificates
+
+# Copy a root ca into the image
+COPY certificates/generated_assets/bacalhau_test_root_ca.crt /usr/local/share/ca-certificates/bacalhau_test_root_ca.crt
+
+# Update CA certificates
+RUN update-ca-certificates
+
+# Download and execute the Bash script from the given URL
+COPY ./bacalhau_bin /usr/local/bin/bacalhau
+RUN chmod +x /usr/local/bin/bacalhau
+
+COPY compute_node_image_setup.sh compute_node_image_setup.sh
+ENTRYPOINT ["/usr/bin/env"]
+CMD ./compute_node_image_setup.sh

test_integration/assets/dockerfiles/Dockerfile-DockerImageRegistryNode

@@ -0,0 +1,24 @@
+FROM registry:2
+
+# Install curl and bash
+RUN apk update && apk add --no-cache curl bash
+
+# Install the ca-certificates package
+RUN apk add --no-cache ca-certificates
+
+# Copy a root ca into the image
+COPY certificates/generated_assets/bacalhau_test_root_ca.crt /usr/local/share/ca-certificates/bacalhau_test_root_ca.crt
+
+# Create a directory to store certificates to be used by the registry
+RUN mkdir /certs
+
+# Copy the certificate and key from the local directory to /certs
+COPY certificates/generated_assets/bacalhau-container-img-registry-node.crt /certs/
+COPY certificates/generated_assets/bacalhau-container-img-registry-node.key /certs/
+
+# Ensure proper permissions for certs
+RUN chmod 600 /certs/bacalhau-container-img-registry-node.key
+RUN chmod 644 /certs/bacalhau-container-img-registry-node.crt
+
+# Expose the registry's default port
+EXPOSE 5000 443

test_integration/assets/dockerfiles/Dockerfile-JumpboxNode

@@ -0,0 +1,28 @@
+# Use the docker:dind image as the base image
+FROM docker:dind
+
+# Set the working directory
+WORKDIR /app
+
+# Install curl and bash
+RUN apk update && apk add --no-cache curl bash
+
+# Install the ca-certificates package
+RUN apk add --no-cache ca-certificates
+
+# Copy a root ca into the image
+COPY certificates/generated_assets/bacalhau_test_root_ca.crt /usr/local/share/ca-certificates/bacalhau_test_root_ca.crt
+
+# Update CA certificates
+RUN update-ca-certificates
+
+# Download and execute the Bash script from the given URL
+COPY ./bacalhau_bin /usr/local/bin/bacalhau
+RUN chmod +x /usr/local/bin/bacalhau
+
+# Download the minio cli binary, make it executable, and move it to /usr/local/bin
+RUN curl -o /tmp/mc https://dl.min.io/client/mc/release/linux-amd64/mc \
+    && chmod +x /tmp/mc \
+    && mv /tmp/mc /usr/local/bin/
+
+ENTRYPOINT ["dockerd-entrypoint.sh"]

test_integration/assets/dockerfiles/Dockerfile-RequesterNode

@@ -0,0 +1,23 @@
+# Use the docker:dind image as the base image
+FROM docker:dind
+
+# Set the working directory
+WORKDIR /app
+
+# Install curl and bash
+RUN apk update && apk add --no-cache curl bash
+
+# Install the ca-certificates package
+RUN apk add --no-cache ca-certificates
+
+# Copy a root ca into the image
+COPY certificates/generated_assets/bacalhau_test_root_ca.crt /usr/local/share/ca-certificates/bacalhau_test_root_ca.crt
+
+# Update CA certificates
+RUN update-ca-certificates
+
+# Download and execute the Bash script from the given URL
+COPY ./bacalhau_bin /usr/local/bin/bacalhau
+RUN chmod +x /usr/local/bin/bacalhau
+
+ENTRYPOINT ["dockerd-entrypoint.sh"]

test_integration/assets/dockerfiles/certificates/README.md

@@ -0,0 +1,9 @@
+# Certificate Generation
+
+The script in the folder allows you to generate certificates that are signed by a root CA, and provide the
+CN and SAN for these leaf certs. The generated certs will be added to the `generated_assets` directory.
+
+Usage: `./generate_leaf_certs.sh <CN_and_SAN>`
+```shell
+./generate_leaf_certs.sh my-bacalhau-requester-node
+```