Merge pull request #181 from python-discord/feat/158/multi-version

Install Multiple Python Versions in the Image

.dockerignore

@@ -9,6 +9,6 @@
 !snekbox/
 !tests/
 !LICENSE
-!NOTICE
+!LICENSE-THIRD-PARTY
 !pyproject.toml
 !README.md

.github/CONTRIBUTING.md

@@ -62,6 +62,14 @@ Updating NsJail mainly involves two steps:
 
 Other things to look out for are breaking changes to NsJail's config format, its command-line interface, or its logging format. Additionally, dependencies may have to be adjusted in the Dockerfile to get a new version to build or run.
 
+## Adding and Updating Python Interpreters
+
+Python interpreters are built using pyenv via the `scripts/build_python.sh` helper script. This script accepts a pyenv version specifier (`pyenv install --list`) and builds the interpreter in a version-specific directory under `/lang/python`. In the image, each minor version of a Python interpreter should have its own build stage and the resulting `/lang/python` directory can be copied from that stage into the `base` stage.
+
+When updating a patch version (e.g. 3.11.3 to 3.11.4), edit the existing build stage in the image for the minor version (3.11); do not add a new build stage. To have access to a new version, pyenv likely needs to be updated. To do so, change the tag in the `git clone` command in the image, but only for the build stage that needs access to the new version. Updating pyenv for all build stages will just cause unnecessary build cache invalidations.
+
+To change the default interpreter used by NsJail, update the target of the `/lang/python/default` symlink created in the `base` stage.
+
 [readme]: ../README.md
 [Dockerfile]: ../Dockerfile
 [Compose v2]: https://docs.docker.com/compose/compose-v2/

Dockerfile

@@ -1,52 +1,73 @@
-# syntax=docker/dockerfile:1
-FROM python:3.11-slim-buster as builder
+# syntax=docker/dockerfile:1.4
+FROM buildpack-deps:buster as builder-nsjail
 
 WORKDIR /nsjail
 
 RUN apt-get -y update \
- && apt-get install -y \
- bison=2:3.3.* \
- flex=2.6.* \
- g++=4:8.3.* \
- gcc=4:8.3.* \
- git=1:2.20.* \
- libprotobuf-dev=3.6.* \
- libnl-route-3-dev=3.4.* \
- make=4.2.* \
- pkg-config=0.29-6 \
- protobuf-compiler=3.6.*
+ && apt-get install -y --no-install-recommends \
+ bison\
+ flex \
+ libprotobuf-dev\
+ libnl-route-3-dev \
+ protobuf-compiler \
+ && rm -rf /var/lib/apt/lists/*
+
 RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \
  && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800
 RUN make
 
+# ------------------------------------------------------------------------------
+FROM buildpack-deps:buster as builder-py-base
+
+ENV PYENV_ROOT=/pyenv \
+ PYTHON_CONFIGURE_OPTS='--disable-test-modules --enable-optimizations \
+ --with-lto --with-system-expat --without-ensurepip'
+
+RUN apt-get -y update \
+ && apt-get install -y --no-install-recommends \
+ libxmlsec1-dev \
+ tk-dev \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY --link scripts/build_python.sh /
+
+# ------------------------------------------------------------------------------
+FROM builder-py-base as builder-py-3_11
+RUN git clone -b v2.3.24 --depth 1 https://github.com/pyenv/pyenv.git $PYENV_ROOT \
+ && /build_python.sh 3.11.4
+
+# ------------------------------------------------------------------------------
+FROM builder-py-base as builder-py-3_12
+RUN git clone -b v2.3.24 --depth 1 https://github.com/pyenv/pyenv.git $PYENV_ROOT \
+ && /build_python.sh 3.12.0rc1
+
 # ------------------------------------------------------------------------------
 FROM python:3.11-slim-buster as base
 
-# Everything will be a user install to allow snekbox's dependencies to be kept
-# separate from the packages exposed during eval.
-ENV PATH=/root/.local/bin:$PATH \
- PIP_DISABLE_PIP_VERSION_CHECK=1 \
- PIP_NO_CACHE_DIR=false \
- PIP_USER=1
+ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \
+ PIP_NO_CACHE_DIR=false
 
 RUN apt-get -y update \
- && apt-get install -y \
- gcc=4:8.3.* \
- git=1:2.20.* \
- libnl-route-3-200=3.4.* \
- libprotobuf17=3.6.* \
+ && apt-get install -y --no-install-recommends \
+ gcc \
+ git \
+ libnl-route-3-200 \
+ libprotobuf17 \
  && rm -rf /var/lib/apt/lists/*
 
-COPY --from=builder /nsjail/nsjail /usr/sbin/
-RUN chmod +x /usr/sbin/nsjail
+COPY --link --from=builder-nsjail /nsjail/nsjail /usr/sbin/
+COPY --link --from=builder-py-3_11 /lang/ /lang/
+COPY --link --from=builder-py-3_12 /lang/ /lang/
+
+RUN chmod +x /usr/sbin/nsjail \
+ && ln -s /lang/python/3.11/ /lang/python/default
 
 # ------------------------------------------------------------------------------
 FROM base as venv
 
-COPY requirements/ /snekbox/requirements/
+COPY --link requirements/ /snekbox/requirements/
 WORKDIR /snekbox
 
-# pip installs to the default user site since PIP_USER is set.
 RUN pip install -U -r requirements/requirements.pip
 
 # This must come after the first pip command! From the docs:
@@ -58,11 +79,12 @@ ARG DEV
 RUN if [ -n "${DEV}" ]; \
  then \
  pip install -U -r requirements/coverage.pip \
- && PYTHONUSERBASE=/snekbox/user_base pip install numpy~=1.19; \
+ && export PYTHONUSERBASE=/snekbox/user_base \
+ && /lang/python/default/bin/python -m pip install --user numpy~=1.19; \
  fi
 
 # At the end to avoid re-installing dependencies when only a config changes.
-COPY config/ /snekbox/config/
+COPY --link config/ /snekbox/config/
 
 ENTRYPOINT ["gunicorn"]
 CMD ["-c", "config/gunicorn.conf.py"]

NOTICE → LICENSE-THIRD-PARTY

@@ -1,9 +1,36 @@
-The Python code at snekbox/config_pb2.py was generated from config.proto in nsjail
-Copyright 2014 Google Inc. All Rights Reserved.
-Copyright 2016 Sergiusz Bazanski. All Rights Reserved.
-
--------------------------------------------------------------------------------
-
+--------------------------------------------------------------------------------
+ MIT License
+Applies to:
+ - Copyright (c) 2014 Docker, Inc.
+ - scripts/build_python.sh: find command for de-bloating Python install
+--------------------------------------------------------------------------------
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+ Apache License, Version 2.0
+Applies to:
+ - Copyright 2014 Google Inc. All Rights Reserved.
+ Copyright 2016 Sergiusz Bazanski. All Rights Reserved.
+ - snekbox/config_pb2.py: generated from config.proto in nsjail
+--------------------------------------------------------------------------------
  Apache License
  Version 2.0, January 2004
  http://www.apache.org/licenses/

README.md

@@ -7,8 +7,7 @@
 
 Python sandbox runners for executing code in isolation aka snekbox.
 
-Supports a memory [virtual read/write file system](#virtual-file-system) within the sandbox,
-allowing text or binary files to be sent and returned.
+Supports a memory [virtual read/write file system](#virtual-file-system) within the sandbox, allowing text or binary files to be sent and returned.
 
 A client sends Python code to a snekbox, the snekbox executes the code, and finally the results of the execution are returned to the client.
 
@@ -100,22 +99,19 @@ Name | Description
 
 ## Third-party Packages
 
-By default, the Python interpreter has no access to any packages besides the
-standard library. Even snekbox's own dependencies like Falcon and Gunicorn are
-not exposed.
+By default, the Python interpreter has no access to any packages besides the standard library. Even snekbox's own dependencies like Falcon and Gunicorn are not exposed.
 
 To expose third-party Python packages during evaluation, install them to a custom user site:
 
 ```sh
-docker exec snekbox /bin/sh -c 'PYTHONUSERBASE=/snekbox/user_base pip install numpy'
+docker exec snekbox /bin/sh -c \
+ 'PYTHONUSERBASE=/snekbox/user_base /lang/python/default/bin/python -m pip install --user numpy'
 ```
 
 In the above command, `snekbox` is the name of the running container. The name may be different and can be checked with `docker ps`.
 
 The packages will be installed to the user site within `/snekbox/user_base`. To persist the installed packages, a volume for the directory can be created with Docker. For an example, see [`docker-compose.yml`].
 
-If `pip`, `setuptools`, or `wheel` are dependencies or need to be exposed, then use the `--ignore-installed` option with pip. However, note that this will also re-install packages present in the custom user site, effectively making caching it futile. Current limitations of pip don't allow it to ignore packages extant outside the installation destination.
-
 ## Development Environment
 
 See [CONTRIBUTING.md](.github/CONTRIBUTING.md).

config/snekbox.cfg

@@ -14,8 +14,10 @@ envar: "OPENBLAS_NUM_THREADS=5"
 envar: "MKL_NUM_THREADS=5"
 envar: "VECLIB_MAXIMUM_THREADS=5"
 envar: "NUMEXPR_NUM_THREADS=5"
-envar: "PYTHONPATH=/snekbox/user_base/lib/python3.11/site-packages"
+envar: "PYTHONDONTWRITEBYTECODE=true"
 envar: "PYTHONIOENCODING=utf-8:strict"
+envar: "PYTHONUNBUFFERED=true"
+envar: "PYTHONUSERBASE=/snekbox/user_base"
 envar: "HOME=home"
 
 keep_caps: false
@@ -79,29 +81,8 @@ mount {
 }
 
 mount {
- src: "/usr/local/lib"
- dst: "/usr/local/lib"
- is_bind: true
- rw: false
-}
-
-mount {
- src: "/usr/local/bin/python"
- dst: "/usr/local/bin/python"
- is_bind: true
- rw: false
-}
-
-mount {
- src: "/usr/local/bin/python3"
- dst: "/usr/local/bin/python3"
- is_bind: true
- rw: false
-}
-
-mount {
- src: "/usr/local/bin/python3.11"
- dst: "/usr/local/bin/python3.11"
+ src: "/lang"
+ dst: "/lang"
  is_bind: true
  rw: false
 }
@@ -116,6 +97,6 @@ cgroup_pids_mount: "/sys/fs/cgroup/pids"
 iface_no_lo: true
 
 exec_bin {
- path: "/usr/local/bin/python"
- arg: "-BSqu"
+ path: "/lang/python/default/bin/python"
+ arg: ""
 }

deployment.yaml

@@ -30,8 +30,9 @@ spec:
  - "/bin/sh"
  - "-c"
  - >-
- PYTHONUSERBASE=/snekbox/user_base
- pip install --user --upgrade
+ find /lang/python -mindepth 1 -maxdepth 1 -type d -exec
+ sh -c 'PYTHONUSERBASE=/snekbox/user_base &&
+ {}/bin/python -m pip install --user -U
  anyio[trio]~=3.6
  arrow~=1.2
  attrs~=22.2
@@ -55,6 +56,7 @@ spec:
  typing-extensions~=4.4
  tzdata~=2022.7
  yarl~=1.8
+ ' \;
  volumes:
  - name: snekbox-user-base-volume
  hostPath:

scripts/build_python.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+shopt -s inherit_errexit
+
+py_version="${1}"
+
+# Install Python interpreter under e.g. /lang/python/3.11/ (no patch version).
+"${PYENV_ROOT}/plugins/python-build/bin/python-build" \
+ "${py_version}" \
+ "/lang/python/${py_version%.*}"
+"/lang/python/${py_version%.*}/bin/python" -m pip install -U pip
+
+# Clean up some unnecessary files to reduce image size bloat.
+find /lang/python/ -depth \
+\( \
+ \( -type d -a \( \
+ -name test -o -name tests -o -name idle_test \
+ \) \) \
+ -o \( -type f -a \( \
+ -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \
+ \) \) \
+\) -exec rm -rf '{}' +

snekbox/nsjail.py

@@ -221,9 +221,9 @@ def python3(
  *nsjail_args,
  "--",
  self.config.exec_bin.path,
- *self.config.exec_bin.arg,
- # Filter out empty strings at start of py_args
+ # Filter out empty strings at start of Python args
  # (causes issues with python cli)
+ *iter_lstrip(self.config.exec_bin.arg),
  *iter_lstrip(py_args),
  ]
 

tests/test_nsjail.py

@@ -79,7 +79,7 @@ def test_subprocess_resource_unavailable(self):
  for _ in range({max_pids}):
  print(subprocess.Popen(
  [
- '/usr/local/bin/python3',
+ '/lang/python/default/bin/python',
  '-c',
  'import time; time.sleep(1)'
  ],
@@ -486,7 +486,7 @@ def test_py_args(self):
  for args, expected in cases:
  with self.subTest(args=args):
  result = self.nsjail.python3(py_args=args)
- idx = result.args.index("-BSqu")
+ idx = result.args.index(self.nsjail.config.exec_bin.path)
  self.assertEqual(result.args[idx + 1 :], expected)
  self.assertEqual(result.returncode, 0)