Add python3.13. (#175)

* Add python3.13. * Fix integ tests --------- Co-authored-by: Stefan Toma <[email protected]>
aws · Nov 15, 2024 · 079135e · 079135e
1 parent 2fe3eca
commit 079135e
Show file tree

Hide file tree

Showing 14 changed files with 253 additions and 12 deletions.
diff --git a/README.md b/README.md
@@ -10,7 +10,7 @@ You can include this package in your preferred base image to make that base imag
 
 ## Requirements
 The Python Runtime Interface Client package currently supports Python versions:
- - 3.7.x up to and including 3.12.x
+ - 3.7.x up to and including 3.13.x
 
 ## Usage
 

diff --git a/awslambdaric/bootstrap.py b/awslambdaric/bootstrap.py
@@ -454,9 +454,10 @@ def run(app_root, handler, lambda_runtime_api_addr):
     sys.stdout = Unbuffered(sys.stdout)
     sys.stderr = Unbuffered(sys.stderr)
 
-    use_thread_for_polling_next = (
-        os.environ.get("AWS_EXECUTION_ENV") == "AWS_Lambda_python3.12"
-    )
+    use_thread_for_polling_next = os.environ.get("AWS_EXECUTION_ENV") in [
+        "AWS_Lambda_python3.12",
+        "AWS_Lambda_python3.13",
+    ]
 
     with create_log_sink() as log_sink:
         lambda_runtime_client = LambdaRuntimeClient(

diff --git a/awslambdaric/lambda_runtime_marshaller.py b/awslambdaric/lambda_runtime_marshaller.py
@@ -15,7 +15,10 @@
 # We also set 'ensure_ascii=False' so that the encoded json contains unicode characters instead of unicode escape sequences
 class Encoder(json.JSONEncoder):
     def __init__(self):
-        if os.environ.get("AWS_EXECUTION_ENV") == "AWS_Lambda_python3.12":
+        if os.environ.get("AWS_EXECUTION_ENV") in {
+            "AWS_Lambda_python3.12",
+            "AWS_Lambda_python3.13",
+        }:
             super().__init__(use_decimal=False, ensure_ascii=False, allow_nan=True)
         else:
             super().__init__(use_decimal=False, allow_nan=True)

diff --git a/setup.py b/setup.py
@@ -91,6 +91,7 @@ def read_requirements(req="base.txt"):
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
         "License :: OSI Approved :: Apache Software License",
         "Operating System :: OS Independent",
     ],

diff --git a/tests/integration/codebuild/buildspec.os.alpine.yml b/tests/integration/codebuild/buildspec.os.alpine.yml
@@ -22,6 +22,7 @@ batch:
             - "3.10"
             - "3.11"
             - "3.12"
+            - "3.13"
 phases:
   pre_build:
     commands:

diff --git a/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml b/tests/integration/codebuild/buildspec.os.amazonlinux.2.yml
@@ -2,7 +2,7 @@ version: 0.2
 
 env:
   variables:
-    OS_DISTRIBUTION: amazonlinux
+    OS_DISTRIBUTION: amazonlinux2
     PYTHON_LOCATION: "/usr/local/bin/python3"
     TEST_NAME: "aws-lambda-python-rtc-amazonlinux-test"
 batch:
@@ -20,7 +20,6 @@ batch:
             - "3.9"
             - "3.10"
             - "3.11"
-            - "3.12"
 phases:
   pre_build:
     commands:

diff --git a/tests/integration/codebuild/buildspec.os.amazonlinux.2023.yml b/tests/integration/codebuild/buildspec.os.amazonlinux.2023.yml
@@ -0,0 +1,105 @@
+version: 0.2
+
+env:
+  variables:
+    OS_DISTRIBUTION: amazonlinux2023
+    PYTHON_LOCATION: "/usr/local/bin/python3"
+    TEST_NAME: "aws-lambda-python-rtc-amazonlinux-test"
+batch:
+  build-matrix:
+    static:
+      ignore-failure: false
+      env:
+        privileged-mode: true
+    dynamic:
+      env:
+        variables:
+          DISTRO_VERSION:
+            - "2023"
+          RUNTIME_VERSION:
+            - "3.12"
+            - "3.13"
+phases:
+  pre_build:
+    commands:
+      - export IMAGE_TAG="python-${OS_DISTRIBUTION}-${DISTRO_VERSION}:${RUNTIME_VERSION}"
+      - echo "Extracting and including the Runtime Interface Emulator"
+      - SCRATCH_DIR=".scratch"
+      - mkdir "${SCRATCH_DIR}"
+      - ARCHITECTURE=$(arch)
+      - >
+        if [[ "$ARCHITECTURE" == "x86_64" ]]; then
+            RIE="aws-lambda-rie"
+        elif [[ "$ARCHITECTURE" == "aarch64" ]]; then
+            RIE="aws-lambda-rie-arm64"
+        else
+            echo "Architecture $ARCHITECTURE is not currently supported."
+            exit 1
+        fi
+      - tar -xvf tests/integration/resources/${RIE}.tar.gz --directory "${SCRATCH_DIR}"
+      - >
+        cp "tests/integration/docker/Dockerfile.echo.${OS_DISTRIBUTION}" \
+          "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp"
+      - >
+        echo "COPY ${SCRATCH_DIR}/${RIE} /usr/bin/${RIE}" >> \
+          "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp"
+      - >
+        echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' > /etc/docker/daemon.json
+        service docker restart
+      - echo "Building image ${IMAGE_TAG}"
+      - >
+        docker build . \
+          -f "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" \
+          -t "${IMAGE_TAG}" \
+          --build-arg RUNTIME_VERSION="${RUNTIME_VERSION}" \
+          --build-arg DISTRO_VERSION="${DISTRO_VERSION}" \
+          --build-arg ARCHITECTURE="${ARCHITECTURE}" \
+          --load
+  build:
+    commands:
+      - set -x
+      - echo "Running Image ${IMAGE_TAG}"
+      - docker network create "${TEST_NAME}-network"
+      - >
+        docker run \
+          --detach \
+          --name "${TEST_NAME}-app" \
+          --network "${TEST_NAME}-network" \
+          --entrypoint="" \
+          "${IMAGE_TAG}" \
+          sh -c "/usr/bin/${RIE} ${PYTHON_LOCATION} -m awslambdaric app.handler"
+      - sleep 2
+      - >
+        docker run \
+          --name "${TEST_NAME}-tester" \
+          --env "TARGET=${TEST_NAME}-app" \
+          --network "${TEST_NAME}-network" \
+          --entrypoint="" \
+          "${IMAGE_TAG}" \
+          sh -c 'curl -X POST "http://${TARGET}:8080/2015-03-31/functions/function/invocations" -d "{}" --max-time 10'
+      - actual="$(docker logs --tail 1 "${TEST_NAME}-tester" | xargs)"
+      - expected='success'
+      - |
+        echo "Response: ${actual}"
+        if [[ "$actual" != "$expected" ]]; then
+          echo "fail! runtime: $RUNTIME - expected output $expected - got $actual"
+          exit -1
+        fi
+    finally:
+      - |
+        echo "---------Container Logs: ${TEST_NAME}-app----------"
+        echo
+        docker logs "${TEST_NAME}-app" || true
+        echo
+        echo "---------------------------------------------------"
+        echo "--------Container Logs: ${TEST_NAME}-tester--------"
+        echo
+        docker logs "${TEST_NAME}-tester" || true
+        echo
+        echo "---------------------------------------------------"
+      - echo "Cleaning up..."
+      - docker stop "${TEST_NAME}-app" || true
+      - docker rm --force "${TEST_NAME}-app" || true
+      - docker stop "${TEST_NAME}-tester" || true
+      - docker rm --force "${TEST_NAME}-tester" || true
+      - docker network rm "${TEST_NAME}-network" || true
diff --git a/tests/integration/codebuild/buildspec.os.debian.yml b/tests/integration/codebuild/buildspec.os.debian.yml
@@ -22,6 +22,7 @@ batch:
             - "3.10"
             - "3.11"
             - "3.12"
+            - "3.13"
 phases:
   pre_build:
     commands:

diff --git a/tests/integration/codebuild/buildspec.os.ubuntu.yml b/tests/integration/codebuild/buildspec.os.ubuntu.yml
@@ -22,6 +22,7 @@ batch:
             - "3.10"
             - "3.11"
             - "3.12"
+            - "3.13"
 phases:
   pre_build:
     commands:

diff --git a/tests/integration/docker/Dockerfile.echo.alpine b/tests/integration/docker/Dockerfile.echo.alpine
@@ -31,6 +31,7 @@ RUN mkdir -p ${RIC_BUILD_DIR}
 # Copy function code and Runtime Interface Client .tgz
 WORKDIR ${RIC_BUILD_DIR}
 COPY . .
+RUN pip3 install setuptools
 RUN make init build test && \
     mv ./dist/awslambdaric-*.tar.gz ./dist/awslambdaric-test.tar.gz
 

diff --git a/...ration/docker/Dockerfile.echo.amazonlinux → ...ation/docker/Dockerfile.echo.amazonlinux2 b/...ration/docker/Dockerfile.echo.amazonlinux → ...ation/docker/Dockerfile.echo.amazonlinux2
@@ -81,10 +81,6 @@ RUN mkdir -p ${RIC_BUILD_DIR}
 WORKDIR ${RIC_BUILD_DIR}
 COPY . .
 
-# distutils no longer available in python3.12 and later
-#   https://docs.python.org/3/whatsnew/3.12.html
-#   https://peps.python.org/pep-0632/
-RUN if [ $(cut -d '.' -f 2 <<< ${RUNTIME_VERSION}) -ge 12 ]; then pip3 install setuptools; fi
 RUN make init build test && \
     mv ./dist/awslambdaric-*.tar.gz ./dist/awslambdaric-test.tar.gz
 

diff --git a/tests/integration/docker/Dockerfile.echo.amazonlinux2023 b/tests/integration/docker/Dockerfile.echo.amazonlinux2023
@@ -0,0 +1,127 @@
+ARG DISTRO_VERSION
+# Stage 1 - bundle base image + runtime interface client
+# Grab a fresh copy of the image and install Python
+FROM public.ecr.aws/amazonlinux/amazonlinux:${DISTRO_VERSION} AS python-amazonlinux-builder
+
+ARG RUNTIME_VERSION
+
+# Install apt dependencies
+RUN dnf install -y \
+  gcc \
+  gcc-c++ \
+  tar \
+  gzip \
+  make \
+  autoconf \
+  automake \
+  freetype-devel \
+  yum-utils \
+  findutils \
+  wget \
+  openssl \
+  openssl-devel \
+  bzip2-devel \
+  libffi-devel \
+  sqlite-devel
+
+RUN RUNTIME_LATEST_VERSION=${RUNTIME_VERSION}.$(curl -s https://www.python.org/ftp/python/ | \
+                                                grep -oE "href=\"$(echo ${RUNTIME_VERSION} | sed "s/\\./\\\./g")\.[0-9]+" | \
+                                                cut -d. -f3 | \
+                                                sort -rn | \
+                                                while read -r patch; do \
+                                                  $(wget -c https://www.python.org/ftp/python/${RUNTIME_VERSION}.$patch/Python-${RUNTIME_VERSION}.$patch.tgz -O Python-${RUNTIME_VERSION}.$patch.tgz); \
+                                                  [ $? -eq 0 ] && echo $patch && break; \
+                                                done) \
+  && tar -xzf Python-${RUNTIME_LATEST_VERSION}.tgz \
+  && cd Python-${RUNTIME_LATEST_VERSION} \
+  && ./configure --prefix=/usr/local --enable-shared \
+  && make \
+  && make install \
+  && ln -s /usr/local/bin/python${RUNTIME_VERSION} /usr/local/bin/python${RUNTIME_LATEST_VERSION}
+
+# Stage 2 - clean python build dependencies
+FROM public.ecr.aws/amazonlinux/amazonlinux:${DISTRO_VERSION} AS python-amazonlinux
+RUN dnf install -y \
+  libffi-devel
+
+# Copy the compiled python to /usr/local
+COPY --from=python-amazonlinux-builder /usr/local /usr/local
+ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
+
+# Stage 3 - build function and dependencies
+FROM python-amazonlinux-builder AS build-image
+ARG RUNTIME_VERSION
+ARG ARCHITECTURE
+
+# Install aws-lambda-cpp build dependencies
+RUN dnf install -y \
+  tar \
+  gzip \
+  make \
+  autoconf \
+  automake \
+  libtool \
+  libcurl-devel \
+  gcc-c++ \
+  wget \
+  sqlite-devel
+
+# Install a modern CMake
+RUN wget --quiet -O cmake-install https://github.com/Kitware/CMake/releases/download/v3.20.0/cmake-3.20.0-linux-${ARCHITECTURE}.sh && \
+    sh cmake-install --skip-license --prefix=/usr --exclude-subdirectory;
+
+ENV PATH=/usr/local/bin:$PATH
+ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
+
+
+# Include global args in this stage of the build
+ARG RIC_BUILD_DIR="/home/build/"
+# Create function directory
+RUN mkdir -p ${RIC_BUILD_DIR}
+# Copy function code and Runtime Interface Client .tgz
+WORKDIR ${RIC_BUILD_DIR}
+COPY . .
+
+# distutils no longer available in python3.12 and later
+#   https://docs.python.org/3/whatsnew/3.12.html
+#   https://peps.python.org/pep-0632/
+RUN pip3 install setuptools
+RUN make init build 
+
+RUN mv ./dist/awslambdaric-*.tar.gz ./dist/awslambdaric-test.tar.gz
+RUN python${RUNTIME_VERSION} -m pip install \
+      ./dist/awslambdaric-test.tar.gz \
+      --target ${RIC_BUILD_DIR}
+
+RUN make test
+
+# Include global args in this stage of the build
+ARG FUNCTION_DIR="/home/app/"
+# Create function directory
+RUN mkdir -p ${FUNCTION_DIR}
+# Copy function code
+COPY tests/integration/test-handlers/echo/* ${FUNCTION_DIR}
+# Copy Runtime Interface Client .tgz
+RUN cp ./dist/awslambdaric-test.tar.gz ${FUNCTION_DIR}/awslambdaric-test.tar.gz
+
+# Install the function's dependencies
+WORKDIR ${FUNCTION_DIR}
+RUN python${RUNTIME_VERSION} -m pip install \
+        awslambdaric-test.tar.gz \
+        --target ${FUNCTION_DIR} && \
+    rm awslambdaric-test.tar.gz
+
+
+# Stage 4 - final runtime interface client image
+# Grab a fresh copy of the Python image
+FROM python-amazonlinux
+RUN dnf install -y brotli
+# Include global arg in this stage of the build
+ARG FUNCTION_DIR="/home/app/"
+# Set working directory to function root directory
+WORKDIR ${FUNCTION_DIR}
+# Copy in the built dependencies
+COPY --from=build-image ${FUNCTION_DIR} ${FUNCTION_DIR}
+
+ENTRYPOINT [ "/usr/local/bin/python3", "-m", "awslambdaric" ]
+CMD [ "app.handler" ]
diff --git a/tests/integration/docker/Dockerfile.echo.debian b/tests/integration/docker/Dockerfile.echo.debian
@@ -19,6 +19,7 @@ RUN mkdir -p ${RIC_BUILD_DIR}
 # Copy function code and Runtime Interface Client .tgz
 WORKDIR ${RIC_BUILD_DIR}
 COPY . .
+RUN pip3 install setuptools
 RUN make init build test && \
     mv ./dist/awslambdaric-*.tar.gz ./dist/awslambdaric-test.tar.gz
 

diff --git a/tests/test_lambda_runtime_marshaller.py b/tests/test_lambda_runtime_marshaller.py
@@ -11,13 +11,17 @@
 
 class TestLambdaRuntimeMarshaller(unittest.TestCase):
     execution_envs = (
+        "AWS_Lambda_python3.13",
         "AWS_Lambda_python3.12",
         "AWS_Lambda_python3.11",
         "AWS_Lambda_python3.10",
         "AWS_Lambda_python3.9",
     )
 
-    envs_lambda_marshaller_ensure_ascii_false = {"AWS_Lambda_python3.12"}
+    envs_lambda_marshaller_ensure_ascii_false = {
+        "AWS_Lambda_python3.12",
+        "AWS_Lambda_python3.13",
+    }
 
     execution_envs_lambda_marshaller_ensure_ascii_true = tuple(
         set(execution_envs).difference(envs_lambda_marshaller_ensure_ascii_false)