Multi stage build docker files

.dockerignore

@@ -0,0 +1,17 @@
+.git
+.vscode
+build/
+build-debug/
+build-release/
+conda-bundle-output/
+test_data/
+*.o
+*.a
+*.so
+*.dylib
+*.exe
+*.tif
+*.shp
+*.dbf
+*.shx
+*.prj

.github/workflows/test.yml

@@ -19,9 +19,6 @@ jobs:
 
     - name: Run tests
       run: |
-        docker run --rm -v $(pwd):/app taudem-linux-run-tests /bin/bash -c "
-          make clean && \
-          make release COMPILER=linux && \
-          make install && \
-          su - taudem-docker -c 'export TAUDEM_PATH=/usr/local/taudem && export PATH=/usr/local/taudem:\$PATH && export OMPI_MCA_rmaps_base_oversubscribe=1 && cd /app && make dk-run-tests'
+        docker run --rm -v $(pwd):/app --user taudem-docker taudem-linux-run-tests /bin/bash -c "
+          export PATH=/usr/local/taudem:\$PATH && export TAUDEM_PATH=/usr/local/taudem && export OMPI_MCA_rmaps_base_oversubscribe=1 && cd /app && make dk-run-tests
         "

Dockerfile

@@ -1,71 +1,62 @@
-# This Dockerfile builds and installs TauDEM on Ubuntu Linux
+# This Dockerfile builds and installs TauDEM using a multi-stage build for optimization
 
-# Using Ubuntu as the base image
-FROM ubuntu:22.04
+# --- Builder Stage ---
+FROM ubuntu:22.04 AS builder
 
 # Avoid interactive prompts during package installation
 ENV DEBIAN_FRONTEND=noninteractive
 
 # Install necessary dependencies to compile TauDEM
-RUN apt update && apt install -y \
+RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential \
     cmake \
-    openmpi-bin \
     libopenmpi-dev \
-    gdal-bin \
     libgdal-dev \
     libproj-dev \
     libtiff-dev \
     libgeotiff-dev \
-    git \
-    nano \
-    sudo \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*  # Clean up to reduce image size
+    && rm -rf /var/lib/apt/lists/*
 
-# Create a non-root user
-RUN useradd -m -s /bin/bash taudem
+# Set working directory
+WORKDIR /src
 
-# Set the password for the taudem user
-RUN echo "taudem:taudem" | chpasswd
+# Copy source code (leveraging .dockerignore to exclude unnecessary files)
+COPY . .
 
-# Grant the taudem user sudo privileges without password
-RUN usermod -aG sudo taudem && \
-    echo "taudem ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
-
-# Create workspace directory
-RUN mkdir -p /home/taudem/workspace && \
-    chown -R taudem:taudem /home/taudem/workspace
-
-# Switch to taudem user
-USER taudem
-WORKDIR /home/taudem/workspace
+# Build and install TauDEM to a temporary directory
+# We use PREFIX=/install so it installs to /install/taudem
+RUN make release COMPILER=linux && \
+    make install PREFIX=/install && \
+    echo "=== Validating build ===" && \
+    test -f /install/taudem/pitremove || (echo "ERROR: pitremove not found" && exit 1)
 
-# Clone the TauDEM repository (clones the Develop branch)
-RUN git clone --depth 1 https://github.com/dtarb/TauDEM.git
+# --- Runtime Stage ---
+FROM ubuntu:22.04
 
-# Set working directory to TauDEM
-WORKDIR /home/taudem/workspace/TauDEM
+LABEL maintainer="TauDEM Developers"
+LABEL description="Parallel C++ suite for terrain analysis using Digital Elevation Models"
 
-# Build and install TauDEM
-RUN make clean && \
-    make release COMPILER=linux && \
-    sudo make install PREFIX=/usr/local && \
-    echo "=== Validating installation ===" && \
-    test -f /usr/local/taudem/pitremove || (echo "ERROR: pitremove not found in /usr/local/taudem" && exit 1) && \
-    echo "SUCCESS: TauDEM installation validated - pitremove found"
+# Avoid interactive prompts
+ENV DEBIAN_FRONTEND=noninteractive
 
-# Set up environment variables - add /usr/local/taudem to PATH
-RUN echo "export PATH=/usr/local/taudem:\$PATH" >> /home/taudem/.bashrc
+# Install only runtime dependencies
+# libgdal30 is the runtime for GDAL on Ubuntu 22.04
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    openmpi-bin \
+    libgdal30 \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
-# Switch back to root for final setup
-USER root
+# Copy the installed TauDEM binaries from the builder stage
+COPY --from=builder /install/taudem /usr/local/taudem
 
 # Set the PATH environment variable globally to include taudem directory
 ENV PATH="/usr/local/taudem:${PATH}"
 
-# Set the working directory
-WORKDIR /home/taudem/workspace/TauDEM
+# Create a non-root user for security
+RUN useradd -m -s /bin/bash taudem
+USER taudem
+WORKDIR /home/taudem
 
 # Set the default command to run an interactive shell
 CMD ["/bin/bash"]

Dockerfile-run.tests

@@ -1,61 +1,88 @@
 # This Dockerfile is for compiling TauDEM for Linux and running the tests. It is not intended for running TauDEM.
 
-# Using Ubuntu as the base image
-FROM ubuntu:22.04
+# --- Builder Stage ---
+FROM ubuntu:22.04 AS builder
 
-# Install necessary dependencies to compile TauDEM
-RUN apt update && apt install -y \
+LABEL maintainer="TauDEM Developers"
+LABEL description="Builder stage for TauDEM test image"
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install build dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential \
     cmake \
-    openmpi-bin \
     libopenmpi-dev \
-    gdal-bin \
     libgdal-dev \
     libproj-dev \
     libtiff-dev \
     libgeotiff-dev \
     git \
+    ca-certificates \
+    sudo \
     nano \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /src
+
+# Copy only source code and essential files (use .dockerignore to exclude build, test_data, etc.)
+COPY . .
+
+# Build TauDEM and install to /install/taudem
+RUN make clean && \
+    make release COMPILER=linux && \
+    make install PREFIX=/install && \
+    test -f /install/taudem/pitremove || (echo "ERROR: pitremove not found" && exit 1)
+
+# --- Test Stage ---
+FROM ubuntu:22.04 AS test
+
+LABEL maintainer="TauDEM Developers"
+LABEL description="Test stage for TauDEM with bats and test data"
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install only runtime and test dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    openmpi-bin \
+    libgdal30 \
+    git \
+    ca-certificates \
+    build-essential \
+    cmake \
     sudo \
-    && rm -rf /var/lib/apt/lists/*  # Clean up to reduce image size
+    nano \
+    && rm -rf /var/lib/apt/lists/*
 
-# Install bats testing framework
-RUN git clone https://github.com/sstephenson/bats.git && \
-    cd bats && \
-    ./install.sh /usr/local && \
+# Install bats testing framework and extensions
+RUN git clone https://github.com/sstephenson/bats.git /tmp/bats && \
+    /tmp/bats/install.sh /usr/local && \
     git clone https://github.com/ztombol/bats-support /tmp/bats-support && \
     git clone https://github.com/ztombol/bats-assert /tmp/bats-assert && \
-    git clone https://github.com/ztombol/bats-file /tmp/bats-file
+    git clone https://github.com/ztombol/bats-file /tmp/bats-file && \
+    rm -rf /tmp/bats
 
-# Create a non-root user
-RUN useradd -m -s /bin/bash taudem-docker
+# Copy TauDEM binaries from builder
+COPY --from=builder /install/taudem /usr/local/taudem
 
-# Set the password for the taudem-docker user (less secure, for demonstration purposes)
-RUN echo "taudem-docker:taudem-docker" | chpasswd
+# Create non-root user for running tests
+RUN useradd -m -s /bin/bash taudem-docker && \
+    echo "taudem-docker:taudem-docker" | chpasswd && \
+    usermod -aG sudo taudem-docker && \
+    echo "export PATH=/usr/local/taudem:\$PATH" >> /home/taudem-docker/.bashrc && \
+    echo "export TAUDEM_PATH=/usr/local/taudem" >> /home/taudem-docker/.bashrc
 
-# Grant the taudem-docker user sudo privileges
-RUN usermod -aG sudo taudem-docker
+# Set up test data directory and permissions
+WORKDIR /home/taudem-docker/workspace
+RUN chown taudem-docker:taudem-docker /home/taudem-docker/workspace
 
-# Create workspace directory for taudem-docker
+# Clone TauDEM-Test-Data as non-root user
+USER taudem-docker
 RUN mkdir -p /home/taudem-docker/workspace/TauDEM-Test-Data
-
-# Clone the TauDEM-Test-Data repo and make the taudem-tests.sh executable for running the tests
-RUN chown taudem-docker:taudem-docker /home/taudem-docker/workspace/TauDEM-Test-Data && \
-    sudo -u taudem-docker git clone https://github.com/dtarb/TauDEM-Test-Data.git /home/taudem-docker/workspace/TauDEM-Test-Data && \
-    chown taudem-docker:taudem-docker /home/taudem-docker/workspace/TauDEM-Test-Data/Input/taudem-tests.sh && \
+RUN git clone https://github.com/dtarb/TauDEM-Test-Data.git /home/taudem-docker/workspace/TauDEM-Test-Data && \
     chmod +x /home/taudem-docker/workspace/TauDEM-Test-Data/Input/taudem-tests.sh
 
-# Create the taudem directory
-RUN mkdir -p /usr/local/taudem
-
-# Set the PATH environment variable for taudem-docker user
-# Setting the TAUDEM_PATH environment variable which is needed only for runinng the tests
-# TAUDEM_PATH is used in the taudem-tests.sh script
-RUN echo "export PATH=/usr/local/taudem:\$PATH" >> /home/taudem-docker/.bashrc && \
-    echo "export TAUDEM_PATH=/usr/local/taudem" >> /home/taudem-docker/.bashrc
-
-# Set the working directory inside the container - this is where we will run commands to build and install TauDEM
 WORKDIR /app
 
-# Set the default command to run an interactive shell
+# Default to interactive shell
 CMD ["/bin/bash"]

conda-bundle/BUILD.md

@@ -30,7 +30,7 @@ Build time: ~5-10 minutes depending on your system.
 
 The build produces:
 
-- `../conda-bundle-output/taudem-conda-linux64.tar.gz` (~352MB) - The portable bundle with all dependencies
+- `../conda-bundle-output/taudem-conda-linux64.tar.gz` (~122MB) - The portable bundle with all dependencies
 
 ## Bundle Structure
 
@@ -100,13 +100,13 @@ To test the bundle in a cloud-based JupyterHub environment:
    which pitremove
 
    # Run a simple command to verify
-   pitremove -h
+   pitremove
    ```
 
-6. **Test with MPI** (if your JupyterHub supports it):
+6. **Test MPI** (if your JupyterHub supports it):
 
    ```bash
-   mpirun -n 2 pitremove -h
+   mpiexec
    ```
 
 **Note**: The bundle is compiled for Linux x86_64. For ARM64 systems, you'll need to modify the Dockerfile to remove the `--platform=linux/amd64` flag.
@@ -128,7 +128,12 @@ To test the bundle in a cloud-based JupyterHub environment:
 ## Technical Details
 
 - Executables are **dynamically linked** to included libraries
-- Libraries included: OpenMPI 5.0.8, GDAL 3.12.0, PROJ 9.7.0, and dependencies
+- Libraries included: GDAL 3.11, PROJ, SQLite, libspatialite, OpenMPI, PMIX, PRTE, and dependencies
+- **OpenMPI, PMIX, and PRTE are fully bundled** - no additional conda install needed
+- All MPI components (mpirun, mpiexec, prte, prterun) are included with help files and configuration
+- CUDA-related plugins are excluded to avoid warnings on CPU-only systems
 - No Python TauDEM tools (pyfiles) are included - executables only
 - Users do not need admin/root privileges to install
 - The `install.sh` script copies binaries and libraries to the active conda environment
+- Binaries are RPATH-patched to find libraries in the conda environment's lib directory
+- Environment variables (OPAL_PREFIX, PRTE_PREFIX, PMIX_PREFIX) are set during conda activation