✨ better dependency management with Poetry (#863)

(rebased to use 0.12.1 cloudcontroller instead of 1.0.0rc4)

* 🔥

* 🎉 replace requirements with pyproject.toml

* Generated lock files

* ✨ script to install all dependencies (for simultaneously testing all submodules)

* ✨ script to generate lock files for each submodule

* ✨ replace pip install requirements with poetry install

* ⬆️ use python 3.12.4

* 🎨

* 🎨 set `package-mode = false`

* 👷 run uvicorn correctly

* 👷 configure dependabot for each submodule

* 👷 install poetry dependencies with writeable cache dir

* 👷 use poetry

* ⏪ .isort.cfg is required

* 👷 use poetry to run pylint

* 👷 copy necessary files for dependency install

* 🎨

* 🎉 configuration file for cloudapi

* 👷 run pytest with poetry

* 🐛 Configure Poetry to not create virtual environments

.dockerignore

@@ -2,8 +2,6 @@
 **
 
 # Except the following
-!requirements.dev.txt
-!requirements.txt
 !app/
 !shared/
 !endorser/

.github/dependabot.yml

@@ -18,3 +18,35 @@ updates:
       interval: weekly
       day: saturday
       time: "07:00"
+
+  # Maintain dependencies for the App submodule
+  - package-ecosystem: pip
+    directory: /app
+    schedule:
+      interval: weekly
+      day: saturday
+      time: "07:00"
+
+  # Maintain dependencies for the Endorser submodule
+  - package-ecosystem: pip
+    directory: /endorser
+    schedule:
+      interval: weekly
+      day: saturday
+      time: "07:00"
+
+  # Maintain dependencies for the Trust Registry submodule
+  - package-ecosystem: pip
+    directory: /trustregistry
+    schedule:
+      interval: weekly
+      day: saturday
+      time: "07:00"
+
+  # Maintain dependencies for Webhooks Service submodule
+  - package-ecosystem: pip
+    directory: /webhooks
+    schedule:
+      interval: weekly
+      day: saturday
+      time: "07:00"

.github/workflows/continuous-deploy.yml

@@ -155,18 +155,19 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - name: Set up Python 3.12.3
+      - name: Set up Python 3.12.4
         uses: actions/setup-python@v5
         with:
-          python-version: 3.12.3
-          cache: pip
-      - name: Install test dependencies.
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt --upgrade
+          python-version: 3.12.4
+      - name: Set up Poetry
+        uses: abatilo/actions-poetry@v2
+        with:
+          poetry-version: "1.8.3"
+      - name: Install dependencies with Poetry
+        run: bash scripts/install_all_dependencies.sh
       - name: Run Pylint
         run: |
-          pylint app/ endorser/ shared/ trustregistry/ webhooks/ --rcfile=.pylintrc -r n --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" --exit-zero > pylintreport.txt
+          poetry run pylint app/ endorser/ shared/ trustregistry/ webhooks/ --rcfile=.pylintrc -r n --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" --exit-zero > pylintreport.txt
 
   test:
     if: github.event.pull_request.draft == false
@@ -192,15 +193,12 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - name: Set up Python 3.12.3
+      - name: Set up Python 3.12.4
         uses: actions/setup-python@v5
         with:
-          python-version: 3.12.3
-          cache: pip
-      - name: Install test dependencies.
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt --upgrade
+          python-version: 3.12.4
+      - name: Install dependencies with Poetry
+        run: bash scripts/install_all_dependencies.sh
       - uses: docker/login-action@v3
         with:
           registry: ghcr.io
@@ -279,16 +277,17 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - name: Set up Python 3.12.3
+      - name: Set up Python 3.12.4
         uses: actions/setup-python@v5
         with:
-          python-version: 3.12.3
+          python-version: 3.12.4
       - name: Install dependencies
         run: pip install coverage
       - name: Download all .coverage artifacts
         uses: actions/download-artifact@v4
         with:
           path: coverage-files
+          pattern: "coverage-*"
       - name: Move coverage files to top-level directory
         run: |
           for dir in coverage-files/coverage-files-*; do

.github/workflows/style-check.yml

@@ -13,10 +13,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set up Python 3.12.3
+      - name: Set up Python 3.12.4
         uses: actions/setup-python@v5
         with:
-          python-version: 3.12.3
+          python-version: 3.12.4
 
       - name: Install dependencies
         run: |

.isort.cfg

@@ -1,3 +1,2 @@
 [settings]
 profile=black
-known_third_party=

README.md

@@ -46,7 +46,7 @@ Without using `make`, you can run:
 Install the requirements:
 
 ```bash
-pip install -r requirements.txt
+bash scripts/install_all_dependencies.sh
 ```
 
 To start the containers and run the tests:

app/pyproject.toml

@@ -0,0 +1,49 @@
+[tool.poetry]
+name = "cloudapi-app"
+version = "3.3.0"
+description = "Main app submodule for cloudapi"
+authors = ["Mourits de Beer <[email protected]>"]
+package-mode = false
+
+[tool.poetry.dependencies]
+python = "^3.12"
+
+aiohttp = "~3.9.2"
+aries-cloudcontroller = "==0.12.1.post0"
+base58 = "~2.1.1"
+fastapi = "~0.111.0"
+fastapi_websocket_pubsub = "~0.3.8"
+httpx = "~0.27.0"
+loguru = "~0.7.2"
+orjson = "~3.10.0"
+pydantic = "~2.8.2"
+pyjwt = "~2.8.0"
+PyYAML = "~6.0"
+typing-extensions = "~4.12.0"
+uvicorn = { version = "~0.30.0", extras = ["standard"] }
+
+[tool.poetry.dev-dependencies]
+anyio = "~4.4.0"
+assertpy = "==1.1"
+black = "~24.4.2"
+isort = "~5.13.2"
+mockito = "~1.5.0"
+pre-commit = "~3.7.0"
+pylint = "~3.2.2"
+pytest = "~8.2.0"
+pytest-cov = "~5.0.0"
+pytest-mock = "~3.14.0"
+
+[build-system]
+requires = ["poetry-core>=1.8.3"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.coverage.run]
+omit = ["tests/*"]
+
+[tool.isort]
+profile = "black"
+
+[tool.pytest.ini_options]
+addopts = "--junitxml=junit.xml -p no:cacheprovider --cov-report=xml --cov-report=term"
+junit_family = "xunit2"

docker-compose.yaml

@@ -513,7 +513,7 @@ services:
       - /bin/sh
       - -c
       - |-
-        pytest --junitxml="/tests/outputs/test_output_$(date +'%Y-%m-%d_%H-%M-%S').xml"
+        poetry run pytest --junitxml="/tests/outputs/test_output_$(date +'%Y-%m-%d_%H-%M-%S').xml"
     environment:
       ACAPY_GOVERNANCE_AGENT_URL: http://governance-ga-agent:3021
       ACAPY_GOVERNANCE_AGENT_API_KEY: adminApiKey
@@ -547,7 +547,7 @@ services:
       - /bin/sh
       - -c
       - |-
-        ./test_runner.sh 5 pytest
+        ./test_runner.sh 5 poetry run pytest
     environment:
       ACAPY_GOVERNANCE_AGENT_URL: http://governance-ga-agent:3021
       ACAPY_GOVERNANCE_AGENT_API_KEY: adminApiKey

dockerfiles/endorser/Dockerfile

@@ -5,10 +5,15 @@ COPY shared /shared
 
 WORKDIR /endorser
 
-RUN pip install --no-cache-dir -r requirements.txt --upgrade
+RUN pip install "poetry==1.8.3"
+# Set Poetry's cache and virtualenvs path to a writable directory
+ENV POETRY_CACHE_DIR=/tmp/.cache/pypoetry
+RUN poetry config cache-dir $POETRY_CACHE_DIR && \
+  poetry config virtualenvs.path $POETRY_CACHE_DIR/virtualenvs && \
+  poetry install
 
 EXPOSE 3009
 
 USER nobody
 
-CMD ["uvicorn", "endorser.main:app", "--reload", "--host", "0.0.0.0", "--port", "3009"]
+CMD ["poetry", "run", "uvicorn", "endorser.main:app", "--reload", "--host", "0.0.0.0", "--port", "3009"]

dockerfiles/fastapi/Dockerfile

@@ -5,10 +5,15 @@ COPY shared /shared
 
 WORKDIR /app
 
-RUN pip install --no-cache-dir -r requirements.txt --upgrade
+RUN pip install "poetry==1.8.3"
+# Set Poetry's cache and virtualenvs path to a writable directory
+ENV POETRY_CACHE_DIR=/tmp/.cache/pypoetry
+RUN poetry config cache-dir $POETRY_CACHE_DIR && \
+  poetry config virtualenvs.path $POETRY_CACHE_DIR/virtualenvs && \
+  poetry install
 
 EXPOSE 8000
 
 USER nobody
 
-CMD ["uvicorn", "app.main:app", "--reload", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["poetry", "run", "uvicorn", "app.main:app", "--reload", "--host", "0.0.0.0", "--port", "8000"]

dockerfiles/tests/Dockerfile

@@ -2,14 +2,21 @@ FROM python:3.12-slim
 
 WORKDIR /tests
 
-COPY requirements.dev.txt .
-COPY requirements.txt .
-COPY app/requirements.txt app/
-COPY endorser/requirements.txt endorser/
-COPY trustregistry/requirements.txt trustregistry/
-COPY webhooks/requirements.txt webhooks/
+# Copy the pyproject.toml for each submodule
+COPY app/pyproject.toml app/
+COPY endorser/pyproject.toml endorser/
+COPY trustregistry/pyproject.toml trustregistry/
+COPY webhooks/pyproject.toml webhooks/
 
-RUN pip install --no-cache-dir -r requirements.txt --upgrade
+# Copy the lock files for each submodule
+COPY app/poetry.lock app/
+COPY endorser/poetry.lock endorser/
+COPY trustregistry/poetry.lock trustregistry/
+COPY webhooks/poetry.lock webhooks/
+
+# Copy and run the install dependencies script
+COPY scripts/install_all_dependencies.sh .
+RUN bash install_all_dependencies.sh
 
 COPY . .
 

dockerfiles/trustregistry/Dockerfile

@@ -5,10 +5,15 @@ COPY shared /shared
 
 WORKDIR /trustregistry
 
-RUN pip install --no-cache-dir -r requirements.txt --upgrade
+RUN pip install "poetry==1.8.3"
+# Set Poetry's cache and virtualenvs path to a writable directory
+ENV POETRY_CACHE_DIR=/tmp/.cache/pypoetry
+RUN poetry config cache-dir $POETRY_CACHE_DIR && \
+  poetry config virtualenvs.path $POETRY_CACHE_DIR/virtualenvs && \
+  poetry install
 
 EXPOSE 8001
 
 USER nobody
 
-CMD ["uvicorn", "trustregistry.main:app", "--reload", "--host", "0.0.0.0", "--port", "8001"]
+CMD ["poetry", "run", "uvicorn", "trustregistry.main:app", "--reload", "--host", "0.0.0.0", "--port", "8001"]

dockerfiles/webhooks/Dockerfile

@@ -5,10 +5,15 @@ COPY shared /shared
 
 WORKDIR /webhooks
 
-RUN pip install --no-cache-dir -r requirements.txt --upgrade
+RUN pip install "poetry==1.8.3"
+# Set Poetry's cache and virtualenvs path to a writable directory
+ENV POETRY_CACHE_DIR=/tmp/.cache/pypoetry
+RUN poetry config cache-dir $POETRY_CACHE_DIR && \
+  poetry config virtualenvs.path $POETRY_CACHE_DIR/virtualenvs && \
+  poetry install
 
 EXPOSE 3010
 
 USER nobody
 
-CMD ["uvicorn", "webhooks.web.main:app", "--reload", "--host", "0.0.0.0", "--port", "3010"]
+CMD ["poetry", "run", "uvicorn", "webhooks.web.main:app", "--reload", "--host", "0.0.0.0", "--port", "3010"]