add test for DinD and DoOD

Dockerfile

@@ -1,16 +1,29 @@
-ARG PYTHON_VERSION
-FROM python:${version}-slim-bookworm
+ARG PYTHON_VERSION=3.10
+FROM python:${PYTHON_VERSION}-slim-bookworm
+
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache
 
 WORKDIR /workspace
 RUN pip install --upgrade pip \
     && apt-get update \
-    && apt-get install -y \
-        freetds-dev \
-    && rm -rf /var/lib/apt/lists/*
+    && apt-get install -y freetds-dev \
+    && apt-get install -y make \
+    # no real need for keeping this image small at the moment
+    && :; # rm -rf /var/lib/apt/lists/*
+
+# install poetry
+RUN bash -c 'python -m venv /opt/poetry-venv && source $_/bin/activate && pip install poetry && ln -s $(which poetry) /usr/bin'
 
-# install requirements we exported from poetry
-COPY build/requirements.txt requirements.txt
-RUN pip install -r requirements.txt
+# install dependencies with poetry
+COPY pyproject.toml .
+COPY poetry.lock .
+RUN poetry install --all-extras --with dev --no-root
 
 # copy project source
 COPY . .
+
+# install project with poetry
+RUN poetry install --all-extras --with dev

Makefile

@@ -32,7 +32,6 @@ lint:  ## Lint all files in the project, which we also run in pre-commit
 	poetry run pre-commit run -a
 
 image: ## Make the docker image for dind tests
-	poetry export -f requirements.txt -o build/requirements.txt
 	docker build --build-arg PYTHON_VERSION=${PYTHON_VERSION} -t ${IMAGE} .
 
 DOCKER_RUN = docker run --rm -v /var/run/docker.sock:/var/run/docker.sock

core/tests/conftest.py

@@ -1,6 +1,35 @@
+from pathlib import Path
+
 import pytest
 from typing import Callable
+import subprocess
 from testcontainers.core.container import DockerClient
+import sys
+
+PROJECT_DIR = Path(__file__).parent.parent.parent.resolve()
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    """
+    Add configuration for custom pytest markers.
+    """
+    config.addinivalue_line(
+        "markers",
+        "inside_docker_check: test used to validate DinD/DooD are working as expected",
+    )
+
+
+@pytest.fixture(scope="session")
+def python_testcontainer_image() -> str:
+    """Build an image with test containers python for DinD and DooD tests"""
+    py_version = ".".join(map(str, sys.version_info[:2]))
+    image_name = f"testcontainers-python:{py_version}"
+    subprocess.run(
+        [*("docker", "build"), *("--build-arg", f"PYTHON_VERSION={py_version}"), *("-t", image_name), "."],
+        cwd=PROJECT_DIR,
+        check=True,
+    )
+    return image_name
 
 
 @pytest.fixture

core/tests/test_docker_in_docker.py

@@ -1,7 +1,19 @@
+import contextlib
+import os
 import time
 import socket
+from pathlib import Path
+from typing import Final
+
+import pytest
+
+from testcontainers.core import utils
+from testcontainers.core.config import testcontainers_config as tcc
+from testcontainers.core.labels import SESSION_ID
+from testcontainers.core.network import Network
 from testcontainers.core.container import DockerContainer
-from testcontainers.core.docker_client import DockerClient
+from testcontainers.core.docker_client import DockerClient, LOGGER
+from testcontainers.core.utils import inside_container
 from testcontainers.core.waiting_utils import wait_for_logs
 
 
@@ -85,3 +97,125 @@ def test_dind_inherits_network():
     not_really_dind.stop()
     not_really_dind.remove()
     custom_network.remove()
+
+
+@contextlib.contextmanager
+def print_surround_header(what: str, header_len: int = 80) -> None:
+    """
+    Helper to visually mark a block with headers
+    """
+    start = f"#   Beginning of {what}"
+    end = f"#   End of {what}"
+
+    print("\n")
+    print("#" * header_len)
+    print(start + " " * (header_len - len(start) - 1) + "#")
+    print("#" * header_len)
+    print("\n")
+
+    yield
+
+    print("\n")
+    print("#" * header_len)
+    print(end + " " * (header_len - len(end) - 1) + "#")
+    print("#" * header_len)
+    print("\n")
+
+
+EXPECTED_NETWORK_VAR: Final[str] = "TCC_EXPECTED_NETWORK"
+
+
+@pytest.mark.inside_docker_check
+@pytest.mark.skipif(not os.environ.get(EXPECTED_NETWORK_VAR), reason="No expected network given")
+def test_find_host_network_in_dood() -> None:
+    """
+    Check that the correct host network is found for DooD
+    """
+    LOGGER.info(f"Running container id={utils.get_running_in_container_id()}")
+    LOGGER.info("Networks: ")
+    assert DockerClient().find_host_network() == os.environ[EXPECTED_NETWORK_VAR]
+
+
+@pytest.mark.skipif(not Path(tcc.ryuk_docker_socket).exists(), reason="No docker socket available")
+def test_dood(python_testcontainer_image: str) -> None:
+    """
+    Run tests marked as inside_docker_check inside docker out of docker
+    """
+
+    docker_sock = tcc.ryuk_docker_socket
+    with Network() as network:
+        with (
+            DockerContainer(
+                image=python_testcontainer_image,
+            )
+            .with_command("poetry run pytest -m inside_docker_check")
+            .with_volume_mapping(docker_sock, docker_sock, "rw")
+            # test also that the correct network was found
+            # but only do this if not already inside a container
+            # as there for some reason this doesn't work
+            .with_env(EXPECTED_NETWORK_VAR, "" if inside_container() else network.name)
+            .with_env("RYUK_RECONNECTION_TIMEOUT", "1s")
+            .with_network(network)
+        ) as container:
+            status = container.get_wrapped_container().wait()
+            stdout, stderr = container.get_logs()
+            # ensure ryuk removed the containers created inside container
+            # because they are bound our network the deletion of the network
+            # would fail otherwise
+            time.sleep(1.1)
+
+    # Show what was done inside test
+    with print_surround_header("test_dood results"):
+        print(stdout.decode("utf-8", errors="replace"))
+        print(stderr.decode("utf-8", errors="replace"))
+    assert status["StatusCode"] == 0
+
+
+def test_dind(python_testcontainer_image: str, tmp_path: Path) -> None:
+    """
+    Run selected tests in Docker in Docker
+    """
+    cert_dir = tmp_path / "certs"
+    dind_name = f"docker_{SESSION_ID}"
+    with Network() as network:
+        with (
+            DockerContainer(image="docker:dind", privileged=True)
+            .with_name(dind_name)
+            .with_volume_mapping(str(cert_dir), "/certs", "rw")
+            .with_env("DOCKER_TLS_CERTDIR", "/certs/docker")
+            .with_env("DOCKER_TLS_VERIFY", "1")
+            .with_network(network)
+            .with_network_aliases("docker")
+        ) as dind_container:
+            wait_for_logs(dind_container, "API listen on")
+            client_dir = cert_dir / "docker" / "client"
+            ca_file = client_dir / "ca.pem"
+            assert ca_file.is_file()
+            try:
+                with (
+                    DockerContainer(image=python_testcontainer_image)
+                    .with_command("poetry run pytest -m inside_docker_check")
+                    .with_volume_mapping(str(cert_dir), "/certs")
+                    # for some reason the docker client does not respect
+                    # DOCKER_TLS_CERTDIR and looks in /root/.docker instead
+                    .with_volume_mapping(str(client_dir), "/root/.docker")
+                    .with_env("DOCKER_TLS_CERTDIR", "/certs/docker/client")
+                    .with_env("DOCKER_TLS_VERIFY", "1")
+                    # docker port is 2376 for https, 2375 for http
+                    .with_env("DOCKER_HOST", "tcp://docker:2376")
+                    .with_network(network)
+                ) as test_container:
+                    status = test_container.get_wrapped_container().wait()
+                    stdout, stderr = test_container.get_logs()
+            finally:
+                # ensure the certs are deleted from inside the container
+                # as they might be owned by root it otherwise could lead to problems
+                # with pytest cleanup
+                dind_container.exec("rm -rf /certs/docker")
+                dind_container.exec("chmod -R a+rwX /certs")
+
+    # Show what was done inside test
+    with print_surround_header("test_dood results"):
+        print(stdout.decode("utf-8", errors="replace"))
+        print(stderr.decode("utf-8", errors="replace"))
+    assert status["StatusCode"] == 0

core/tests/test_ryuk.py

@@ -11,11 +11,14 @@
 from testcontainers.core.waiting_utils import wait_for_logs
 
 
+@pytest.mark.inside_docker_check
 def test_wait_for_reaper(monkeypatch: MonkeyPatch):
     Reaper.delete_instance()
     monkeypatch.setattr(testcontainers_config, "ryuk_reconnection_timeout", "0.1s")
-    docker_client = DockerClient()
-    container = DockerContainer("hello-world").start()
+    container = DockerContainer("hello-world")
+    container.start()
+
+    docker_client = container.get_docker_client().client
 
     container_id = container.get_wrapped_container().short_id
     reaper_id = Reaper._container.get_wrapped_container().short_id
@@ -38,6 +41,7 @@ def test_wait_for_reaper(monkeypatch: MonkeyPatch):
     Reaper.delete_instance()
 
 
+@pytest.mark.inside_docker_check
 def test_container_without_ryuk(monkeypatch: MonkeyPatch):
     Reaper.delete_instance()
     monkeypatch.setattr(testcontainers_config, "ryuk_disabled", True)
@@ -46,6 +50,7 @@ def test_container_without_ryuk(monkeypatch: MonkeyPatch):
         assert Reaper._instance is None
 
 
+@pytest.mark.inside_docker_check
 def test_ryuk_is_reused_in_same_process():
     with DockerContainer("hello-world") as container:
         wait_for_logs(container, "Hello from Docker!")

modules/mysql/tests/test_mysql.py

@@ -9,6 +9,7 @@
 from testcontainers.mysql import MySqlContainer
 
 
+@pytest.mark.inside_docker_check
 def test_docker_run_mysql():
     config = MySqlContainer("mysql:8.3.0")
     with config as mysql:

modules/postgres/tests/test_postgres.py

@@ -28,6 +28,7 @@ def fail(*args, **kwargs):
         assert status == 0
 
 
+@pytest.mark.inside_docker_check
 def test_docker_run_postgres_with_sqlalchemy():
     postgres_container = PostgresContainer("postgres:9.5")
     with postgres_container as postgres:

pyproject.toml

@@ -201,6 +201,9 @@ line-length = 120
 addopts = "--tb=short --strict-markers"
 log_cli = true
 log_cli_level = "INFO"
+markers = [
+    "inside_docker_check: mark test to be used to validate DinD/DooD is working as expected"
+]
 
 [tool.coverage.run]
 branch = true