Support running Bacalhau in Docker compose (#4596)

This pull requests addresses issue #4595

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
   - id: detect-aws-credentials
     args: [--allow-missing-credentials]
   - id: detect-private-key
-    exclude: testdata/.*
+    exclude: 'testdata/.*|test-integration/certificates/.*'
   - id: check-yaml
   - id: check-json
 - repo: https://github.com/astral-sh/ruff-pre-commit

test-integration/Dockerfile-ClientNode

@@ -0,0 +1,27 @@
+# Use the docker:dind image as the base image
+FROM docker:dind
+
+# Set the working directory
+WORKDIR /app
+
+# Install curl and bash
+RUN apk update && apk add --no-cache curl bash
+
+# Install the ca-certificates package
+RUN apk add --no-cache ca-certificates
+
+# Copy a root ca into the image
+COPY certificates/generated_assets/bacalhau_test_root_ca.crt /usr/local/share/ca-certificates/bacalhau_test_root_ca.crt
+
+# Update CA certificates
+RUN update-ca-certificates
+
+# Download and execute the Bash script from the given URL
+RUN curl -sSL https://get.bacalhau.org/install.sh | bash
+
+# Download the binary, make it executable, and move it to /usr/local/bin
+RUN curl -o /tmp/mc https://dl.min.io/client/mc/release/linux-amd64/mc \
+    && chmod +x /tmp/mc \
+    && mv /tmp/mc /usr/local/bin/
+
+ENTRYPOINT ["dockerd-entrypoint.sh"]

test-integration/Dockerfile-ComputeNode

@@ -0,0 +1,24 @@
+# Use the docker:dind image as the base image
+FROM docker:dind
+
+# Set the working directory
+WORKDIR /app
+
+# Install curl and bash
+RUN apk update && apk add --no-cache curl bash
+
+# Install the ca-certificates package
+RUN apk add --no-cache ca-certificates
+
+# Copy a root ca into the image
+COPY certificates/generated_assets/bacalhau_test_root_ca.crt /usr/local/share/ca-certificates/bacalhau_test_root_ca.crt
+
+# Update CA certificates
+RUN update-ca-certificates
+
+# Download and execute the Bash script from the given URL
+RUN curl -sSL https://get.bacalhau.org/install.sh | bash
+
+COPY compute_node_image_setup.sh compute_node_image_setup.sh
+ENTRYPOINT ["/usr/bin/env"]
+CMD ./compute_node_image_setup.sh

test-integration/Dockerfile-DockerImageRegistryNode

@@ -0,0 +1,24 @@
+FROM registry:2
+
+# Install curl and bash
+RUN apk update && apk add --no-cache curl bash
+
+# Install the ca-certificates package
+RUN apk add --no-cache ca-certificates
+
+# Copy a root ca into the image
+COPY certificates/generated_assets/bacalhau_test_root_ca.crt /usr/local/share/ca-certificates/bacalhau_test_root_ca.crt
+
+# Create a directory to store certificates to be used by the registry
+RUN mkdir /certs
+
+# Copy the certificate and key from the local directory to /certs
+COPY certificates/generated_assets/bacalhau-container-img-registry-node.crt /certs/
+COPY certificates/generated_assets/bacalhau-container-img-registry-node.key /certs/
+
+# Ensure proper permissions for certs
+RUN chmod 600 /certs/bacalhau-container-img-registry-node.key
+RUN chmod 644 /certs/bacalhau-container-img-registry-node.crt
+
+# Expose the registry's default port
+EXPOSE 5000 443

test-integration/Dockerfile-RequesterNode

@@ -0,0 +1,22 @@
+# Use the docker:dind image as the base image
+FROM docker:dind
+
+# Set the working directory
+WORKDIR /app
+
+# Install curl and bash
+RUN apk update && apk add --no-cache curl bash
+
+# Install the ca-certificates package
+RUN apk add --no-cache ca-certificates
+
+# Copy a root ca into the image
+COPY certificates/generated_assets/bacalhau_test_root_ca.crt /usr/local/share/ca-certificates/bacalhau_test_root_ca.crt
+
+# Update CA certificates
+RUN update-ca-certificates
+
+# Download and execute the Bash script from the given URL
+RUN curl -sSL https://get.bacalhau.org/install.sh | bash
+
+ENTRYPOINT ["dockerd-entrypoint.sh"]

test-integration/README.md

@@ -0,0 +1,198 @@
+# Running Bacalhau on Docker
+
+## Overview
+
+Since Bacalhau is a distributed system with multiple components, it is critical to have a reliable method for end-to-end testing. Additionally, it's important that these tests closely resemble a real production environment without relying on mocks.
+
+This setup addresses those needs by running Bacalhau inside containers while also supporting Docker workloads within these containers (using Docker-in-Docker, or DinD).
+
+## Architecture
+
+- A Requester Docker container, running Bacalhau as a requester node.
+- A Compute Docker container, running Bacalhau as a compute node and is configured to run Docker containers inside it.
+- A Bacalhau Client Docker container to act as a jumpbox to interact with this Bacalhau deployment.
+- A [Registry](https://github.com/distribution/distribution/) Docker container to act as the local container image registry.
+- A Minio Docker container to support running S3 compatible input/output jobs.
+- Docker Compose is used to create 5 services: the Requester Node, the Compute Node, the Client CLI Node, the registry node, and the Minio node.
+- All the services are connected on the same Docker network, allowing them to communicate over the bridged network.
+- All the containers have an injected custom Certificate Authority, which is used for a portion of the internal TLS communication.
+  - TODO: Expand the TLS setup to more components. Now it is used for the registry communication only.
+
+## Setup
+
+---
+### Build the Docker Images
+
+Build the Requester Node image:
+```shell
+docker build -f Dockerfile-RequesterNode -t bacalhau-requester-node-image .
+```
+
+Build the Compute Node image:
+```shell
+docker build -f Dockerfile-ComputeNode -t bacalhau-compute-node-image .
+```
+
+Build the Client Node image:
+```shell
+docker build -f Dockerfile-ClientNode -t bacalhau-client-node-image .
+```
+
+Build the Registry Node image:
+```shell
+docker build -f Dockerfile-DockerImageRegistryNode -t bacalhau-container-img-registry-node-image .
+```
+
+After running these commands, you should see the above images created:
+```shell
+docker image ls
+```
+---
+### Running the setup
+
+Run Docker Compose:
+```shell
+docker-compose up
+```
+
+Access the utility client container to use the Bacalhau CLI:
+```shell
+docker exec -it bacalhau-client-node-container /bin/bash
+```
+
+Once inside the container, you can run the following commands to verify the setup:
+```shell
+# You should see two nodes: a Requestor and a Compute Node
+bacalhau node list
+```
+
+Run a test workload
+```shell
+bacalhau docker run hello-world
+
+# Describe the job; it should have completed successfully.
+bacalhau job describe ........
+```
+
+In another terminal window, you can follow the logs of the Requester node, and compute node
+```shell
+docker logs bacalhau-requester-node-container -f
+docker logs bacalhau-compute-node-container -f
+```
+
+---
+### Setting Up Minio
+
+Access the utility client container to use the Bacalhau CLI:
+```shell
+docker exec -it bacalhau-client-node-container /bin/bash
+```
+
+Setup an alias for the Minio CLI
+```shell
+# The environment variables are already injected in
+# the container, no need to replce them yourself.
+mc alias set bacalhau-minio "http://${BACALHAU_MINIO_NODE_HOST}:9000" "${MINIO_ROOT_USER}" "${MINIO_ROOT_PASSWORD}"
+mc admin info bacalhau-minio
+```
+
+Create a bucket and add some files
+```shell
+mc mb bacalhau-minio/my-data-bucket
+mc ls bacalhau-minio/my-data-bucket/section1/
+echo "This is a sample text hello hello." > example.txt
+mc cp example.txt bacalhau-minio/my-data-bucket/section1/
+```
+
+RUn a job with data input from the minion bucket
+
+```shell
+# Content of aws-test-job.yaml below
+bacalhau job run aws-test-job.yaml
+```
+
+```yaml
+Name: S3 Job Data Access Test
+Type: batch
+Count: 1
+Tasks:
+  - Name: main
+    Engine:
+      Type: docker
+      Params:
+        Image: ubuntu:latest
+        Entrypoint:
+          - /bin/bash
+        Parameters:
+          - "-c"
+          - "cat /put-my-s3-data-here/example.txt"
+    InputSources:
+      - Target: "/put-my-s3-data-here"
+        Source:
+          Type: s3
+          Params:
+            Bucket: "my-data-bucket"
+            Key: "section1/"
+            Endpoint: "http://bacalhau-minio-node:9000"
+            Region: "us-east-1" # If no region added, it fails, even for minio
+```
+
+---
+### Setting Up private registry
+
+This docker compose deployment has a private registry deployed on its own node. It allows us to
+create tests and experiment with docker images jobs without the need to use DockerHub in anyway.
+
+From inside the client container, let's pull an image from DockerHub, push it to our own private registry,
+then run a docker job running the image in out private registry.
+
+```shell
+# pull from docker hub
+docker pull ubuntu
+
+# tag the image to prepare it to be push to our private registry
+docker image tag ubuntu bacalhau-container-img-registry-node:5000/firstbacalhauimage
+
+# push the image to our private registry
+docker push bacalhau-container-img-registry-node:5000/firstbacalhauimage
+```
+
+Now, let's create a job that references that image in private registry:
+
+```shell
+# Content of private-registry-test-job.yaml below
+bacalhau job run private-registry-test-job.yaml
+```
+
+```yaml
+Name: Job to test using local registry images
+Type: batch
+Count: 1
+Tasks:
+  - Name: main
+    Engine:
+      Type: docker
+      Params:
+        Image: bacalhau-container-img-registry-node:5000/firstbacalhauimage
+        Entrypoint:
+          - /bin/bash
+        Parameters:
+          - "-c"
+          - "echo test-local-registry"
+```
+
+---
+### Notes:
+
+If for some reason after running `docker-compose up`, you faced issues with the Image registry node starting, try to remove the image registry docker volume by running:
+
+```shell
+# Destroy the deployment
+docker-compose down
+
+# Remove registry volume
+docker volume rm test-integration_registry-volume
+
+# Create deployment again
+docker-compose up
+```

test-integration/certificates/README.md

@@ -0,0 +1,9 @@
+# Certificate Generation
+
+The script in the folder allows you to generate certificates that are signed by a root CA, and provide the
+CN and SAN for these leaf certs. The generated certs will be added to the `generated_assets` directory.
+
+Usage: `./generate_leaf_certs.sh <CN_and_SAN>`
+```shell
+./generate_leaf_certs.sh my-bacalhau-requester-node
+```

test-integration/certificates/generate_leaf_certs.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+# Set variables
+ROOT_CA_CERT="generated_assets/bacalhau_test_root_ca.crt"
+ROOT_CA_KEY="generated_assets/bacalhau_test_root_ca.key"
+DAYS_VALID=1825  # 5 years
+
+# Organization name and country (same as before)
+ORG_NAME="Bacalhau"
+COUNTRY="US"
+
+# Check if the input argument is provided
+if [[ -z "$1" ]]; then
+    echo "Error: Please provide a string for the Common Name and Subject Alternative Names."
+    exit 1
+fi
+
+COMMON_NAME="$1"
+OUTPUT_CERT="generated_assets/${COMMON_NAME}.crt"
+OUTPUT_KEY="generated_assets/${COMMON_NAME}.key"
+CSR_PATH="generated_assets/${COMMON_NAME}.csr"
+CNF_PATH="generated_assets/${COMMON_NAME}.cnf"
+
+# Check if the files already exist
+if [[ -f "${OUTPUT_CERT}" ]] || [[ -f "${OUTPUT_KEY}" ]]; then
+    echo "Error: One or both of the following files already exist:"
+    [[ -f "${OUTPUT_CERT}" ]] && echo " - ${OUTPUT_CERT}"
+    [[ -f "${OUTPUT_KEY}" ]] && echo " - ${OUTPUT_KEY}"
+    echo "Please remove or rename the existing files before running this script."
+    exit 1
+fi
+
+# Generate a private key for the new certificate
+echo "Generating certificate signed by the root CA..."
+openssl genpkey -algorithm RSA -out "${OUTPUT_KEY}" -pkeyopt rsa_keygen_bits:4096
+
+# Create an OpenSSL configuration file for the SAN
+cat > "${CNF_PATH}" <<EOF
+[ req ]
+default_bits       = 4096
+distinguished_name = req_distinguished_name
+req_extensions     = v3_req
+prompt             = no
+
+[ req_distinguished_name ]
+CN = ${COMMON_NAME}
+O = ${ORG_NAME}
+C = ${COUNTRY}
+
+[ v3_req ]
+keyUsage = critical, digitalSignature, cRLSign, keyEncipherment
+extendedKeyUsage = serverAuth, clientAuth
+subjectAltName = @alt_names
+
+[ alt_names ]
+DNS.1 = ${COMMON_NAME}
+EOF
+
+# Generate a certificate signing request (CSR) using the config file
+openssl req -new -key "${OUTPUT_KEY}" -out "${CSR_PATH}" -config "${CNF_PATH}"
+
+# Sign the certificate with the root CA
+openssl x509 -req -in "${CSR_PATH}" -CA "${ROOT_CA_CERT}" -CAkey "${ROOT_CA_KEY}" \
+    -out "${OUTPUT_CERT}" -days "${DAYS_VALID}" -sha256 -extensions v3_req -extfile "${CNF_PATH}"
+
+# Clean up the CSR and config file
+rm "${CSR_PATH}" "${CNF_PATH}"
+
+echo "Certificate generated and saved to ${OUTPUT_CERT} and ${OUTPUT_KEY}"
+
+echo "Done!"