Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add circleci tests for multiple python versions
Browse files Browse the repository at this point in the history
- Add matrix jobs to unit and ftp/sftp functional tests for python versions 3.9-3.11
- Update circleci cache keys to be compatible with matrix jobs
- Update dependencies to avoid import erros in python versions 3.10 and 3.11
- Fix black version in Pipfile to avoid lint and format clash
- Update deprecated moto mock call in conftest.py
STMilligan committed Apr 30, 2024
1 parent e6e2a09 commit c50668a
Showing 4 changed files with 1,255 additions and 724 deletions.
114 changes: 84 additions & 30 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -8,8 +8,13 @@ version: 2.1

executors:
standard:
parameters:
version:
description: "python version"
default: "3.9"
type: string
docker:
- image: circleci/python:3.9
- image: cimg/python:<< parameters.version >>
environment:
PIPENV_VENV_IN_PROJECT: true
working_directory: "~/lib"
@@ -42,54 +47,79 @@ commands:
- "./.git"

update_virtualenv:
parameters:
version:
description: "python version"
default: "3.9"
type: string
steps:
- restore_cache:
keys:
- virtualenv-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
- virtualenv-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-
- virtualenv-{{ .Environment.CACHE_VERSION }}-
- virtualenv-<< parameters.version >>{{ .Environment.CACHE_VERSION }}-{{ .Branch }}
- virtualenv-<< parameters.version >>{{ .Environment.CACHE_VERSION }}
- configure_github
- run:
name: Install Python packages
command: make sync
- save_cache:
key: virtualenv-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
key: virtualenv-<< parameters.version >>{{ .Environment.CACHE_VERSION }}-{{ .Branch }}
paths:
- "./.venv"

jobs:
lint:
executor: standard
parameters:
version:
description: "python version"
default: "3.9"
type: string
executor:
name: standard
version: << parameters.version >>
steps:
- checkout_source
- update_virtualenv
- update_virtualenv:
version: << parameters.version >>
- run:
name: Run Python linter
command: make lint

unit:
executor: standard
parameters:
version:
description: "python version"
default: "3.9"
type: string
executor:
name: standard
version: << parameters.version >>
steps:
- checkout_source
- update_virtualenv
- update_virtualenv:
version: << parameters.version >>
- run:
name: Run Python unit tests
command: make unit

release:
parameters:
version:
description: "python version"
type: string
default: "3.9"
working_directory: ~/lib

docker:
- image: circleci/python:3.9
- image: cimg/python:<< parameters.version >>

steps:
- checkout_source
- update_virtualenv
- update_virtualenv:
version: "3.9"
- restore_cache:
keys:
- pip-cache-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
- pip-cache-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-
- pip-cache-{{ .Environment.CACHE_VERSION }}-
- virtualenv-<< parameters.version >>{{ .Environment.CACHE_VERSION }}-{{ .Branch }}
- virtualenv-<< parameters.version >>{{ .Environment.CACHE_VERSION }}

- run:
name: verify git tag vs. version
@@ -111,10 +141,15 @@ jobs:
functional-ftp:
parameters:
version:
description: "python version"
type: string
default: "3.9"
working_directory: ~/lib

docker:
- image: circleci/python:3.9
- image: cimg/python:<< parameters.version >>
environment:
TENTACLIO__CONN__FTP_TEST: ftp://octopus:tentacle@localhost

@@ -128,18 +163,25 @@ jobs:

- restore_cache:
keys:
- pip-cache-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
- virtualenv-<< parameters.version >>{{ .Environment.CACHE_VERSION }}-{{ .Branch }}
- virtualenv-<< parameters.version >>{{ .Environment.CACHE_VERSION }}

- run:
name: Install Python packages
command: make sync



functional-sftp:
parameters:
version:
description: "python version"
type: string
default: "3.9"
working_directory: ~/lib

docker:
- image: circleci/python:3.9
- image: cimg/python:<< parameters.version >>
environment:
TENTACLIO__CONN__SFTP_TEST: sftp://octopus:tentacle@localhost:22

@@ -151,7 +193,8 @@ jobs:

- restore_cache:
keys:
- pip-cache-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
- virtualenv-<< parameters.version >>{{ .Environment.CACHE_VERSION }}-{{ .Branch }}
- virtualenv-<< parameters.version >>{{ .Environment.CACHE_VERSION }}

- run:
name: Install Python packages
@@ -163,7 +206,6 @@ jobs:


workflows:
version: 2

test-build-deploy:
jobs:
@@ -173,23 +215,35 @@ workflows:
only: /.*/

- unit:
filters:
tags:
only: /.*/
name: unit-<< matrix.version >>
matrix:
parameters:
version: ["3.9","3.10","3.11"]
filters:
tags:
only: /.*/
- functional-ftp:
filters:
tags:
only: /.*/
name: functional-ftp-<< matrix.version >>
matrix:
parameters:
version: ["3.9","3.10","3.11"]
filters:
tags:
only: /.*/
requires:
- lint
- unit
- unit-<< matrix.version >>
- functional-sftp:
filters:
tags:
only: /.*/
name: functional-sftp-<< matrix.version >>
matrix:
parameters:
version: ["3.9","3.10","3.11"]
filters:
tags:
only: /.*/
requires:
- lint
- unit
- unit-<< matrix.version >>
- release:
requires:
- functional-ftp
7 changes: 2 additions & 5 deletions Pipfile
Original file line number Diff line number Diff line change
@@ -3,14 +3,11 @@ url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"

[requires]
python_version = "3.9"

[dev-packages]
# Symlink to project root
tentaclio = {editable = true,path = "."}
# Linting
black = "*"
black = "==23.9.1"
isort = "*"
flake8 = "*"
mypy = "*"
@@ -19,10 +16,10 @@ types-paramiko = "*"
types-pyyaml = "*"
types-requests = "*"
# Testing
moto = "*"
pytest = "*"
pytest-cov = "*"
pytest-mock = "*"
# Releasing
twine = "*"
secretstorage = "*"
moto = "*"
1,856 changes: 1,168 additions & 688 deletions Pipfile.lock

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion tests/conftest.py
Original file line number Diff line number Diff line change
@@ -38,7 +38,7 @@ def postgres_url():
@pytest.fixture(scope="function")
def s3_client(s3_url):
"""Function level fixture due to cumbersome way of deleting non-empty AWS buckets"""
with moto.mock_s3():
with moto.mock_aws():
with clients.S3Client(s3_url) as client:
yield client

0 comments on commit c50668a

Please sign in to comment.