Skip to content
This repository has been archived by the owner on Nov 4, 2024. It is now read-only.

build: add python 3.11 and 3.12 ci checks #4153

Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 30 additions & 18 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,59 +11,71 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
include:
- django-env: django32
testname: quality-and-jobs
targets: PYTHON_ENV=py38 requirements.js check_translations_up_to_date validate_translations clean_static static quality validate_js check_keywords
- django-env: django32
testname: test-python
targets: PYTHON_ENV=py38 requirements.js clean_static static validate_python
- django-env: django32
testname: acceptance-python
targets: PYTHON_ENV=py38 requirements.js clean_static static acceptance

python-version: ['py38', 'py311', 'py312']
django-env: ['django32']
test: ['acceptance-python', 'test-python', 'quality-and-jobs']
steps:
- uses: actions/checkout@v2
- name: Setup and Format Python Version
id: format_python_version
shell: bash
run: |
# Remove 'py' and insert a dot to format the version
FORMATTED_VERSION=${{ matrix.python-version }} # e.g., py38
FORMATTED_VERSION=${FORMATTED_VERSION/py3/3.} # becomes 3.8
# Set environment variables
echo "PYTHON_VERSION=$FORMATTED_VERSION" >> $GITHUB_ENV
- name: Start container
run: |
docker-compose -f ./.ci/docker-compose-ci.yml up -d
docker compose -f ./.ci/docker-compose-ci.yml up -d
docker exec ecommerce_testing bash -c "
sudo apt-get update -y &&
sudo apt-get install python$PYTHON_VERSION \
python$PYTHON_VERSION-dev \
python$PYTHON_VERSION-distutils \
default-libmysqlclient-dev build-essential pkg-config -y &&
curl -sS https://bootstrap.pypa.io/get-pip.py | python$PYTHON_VERSION;"
# Need to install pip from source here^ otherwise some packages don't get installed
- name: Install dependencies
run: |
docker exec -t ecommerce_testing bash -c "
cd /edx/app/ecommerce/ecommerce/ &&
python3 -m pip install tox
python$PYTHON_VERSION -m pip install tox
"
- name: Run tests
run: |
docker exec -t -e CI=1 ecommerce_testing bash -c "
cd /edx/app/ecommerce/ecommerce/ &&
PATH=\$PATH:/edx/app/ecommerce/nodeenvs/ecommerce/bin:/snap/bin
DJANGO_ENV=${{ matrix.django-env }} make ${{ matrix.targets }}
DJANGO_ENV=${{ matrix.django-env }} PYTHON_ENV=${{ matrix.python-version }} PYTHON_VERSION=$PYTHON_VERSION make ${{ matrix.test }}
"
- name: Run coverage
if: matrix.testname == 'test-python'
if: matrix.test == 'test-python'
run: |
docker exec ecommerce_testing /edx/app/ecommerce/ecommerce/.ci/run_coverage.sh
- name: Setup Python
if: matrix.testname == 'test-python'
if: matrix.test == 'test-python' && matrix.python-version == 'py38'
uses: actions/setup-python@v2
with:
python-version: "3.8"
architecture: x64
- name: Report coverage
if: matrix.testname == 'test-python'
if: matrix.test == 'test-python' && matrix.python-version == 'py38'
uses: codecov/codecov-action@v3
with:
flags: unittests
fail_ci_if_error: false

docs:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ['3.8', '3.11', '3.12']
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
python-version: "3.8"
python-version: ${{matrix.python-version}}
architecture: x64
- name: Install Dependencies
run: pip install -r requirements/docs.txt -r requirements/tox.txt
Expand Down
51 changes: 29 additions & 22 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
NODE_BIN=./node_modules/.bin
DIFF_COVER_BASE_BRANCH=master
PYTHON_ENV=py38
PYTHON_ENV_VAR=$(if $(PYTHON_ENV),$(PYTHON_ENV),py312)
DJANGO_ENV_VAR=$(if $(DJANGO_ENV),$(DJANGO_ENV),django32)
PYTHON_VERSION_VAR=$(if $(PYTHON_VERSION),$(PYTHON_VERSION),3.12)

help:
@echo ''
Expand Down Expand Up @@ -45,17 +46,17 @@ requirements: requirements.js
pip3 install -r requirements/dev.txt --exists-action w

requirements.tox:
pip3 install -U pip==20.0.2
pip3 install -U pip
pip3 install -r requirements/tox.txt --exists-action w

production-requirements: requirements.js
pip3 install -r requirements.txt --exists-action w

migrate: requirements.tox
tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-migrate
python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-migrate

serve: requirements.tox
tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-serve
python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-serve

clean:
find . -name '*.pyc' -delete
Expand All @@ -65,18 +66,18 @@ clean_static:
rm -rf assets/* ecommerce/static/build/*

run_check_isort: requirements.tox
tox -e $(PYTHON_ENV)-check_isort
python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-check_isort

run_isort: requirements.tox
tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-run_isort
python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-run_isort

run_pycodestyle: requirements.tox
tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-pycodestyle
python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-pycodestyle

run_pep8: run_pycodestyle

run_pylint: requirements.tox
tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-pylint
python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-pylint

quality: run_check_isort run_pycodestyle run_pylint

Expand All @@ -86,42 +87,42 @@ validate_js:
$(NODE_BIN)/gulp lint

validate_python: clean requirements.tox
tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-tests
python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-tests

acceptance: clean requirements.tox
tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-acceptance
python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-acceptance

fast_validate_python: clean requirements.tox
DISABLE_ACCEPTANCE_TESTS=True tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-tests
DISABLE_ACCEPTANCE_TESTS=True python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-tests

validate: validate_python validate_js quality

theme_static: requirements.tox
tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-theme_static
python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-theme_static

static: requirements.js theme_static requirements.tox
$(NODE_BIN)/r.js -o build.js
tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-static
python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-static

html_coverage: requirements.tox
tox -e $(PYTHON_ENV)-coverage_html
python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-coverage_html

diff_coverage: validate fast_diff_coverage

fast_diff_coverage: requirements.tox
tox -e $(PYTHON_ENV)-fast_diff_coverage
python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-fast_diff_coverage

e2e: requirements.tox
tox -e $(PYTHON_ENV)-e2e
python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-e2e

extract_translations: requirements.tox
tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-extract_translations
python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-extract_translations

dummy_translations: requirements.tox
tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-dummy_translations
python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-dummy_translations

compile_translations: requirements.tox
tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-compile_translations
python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-compile_translations

fake_translations: extract_translations dummy_translations compile_translations

Expand All @@ -134,18 +135,18 @@ update_translations: pull_translations fake_translations

# extract_translations should be called before this command can detect changes
detect_changed_source_translations: requirements.tox
tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-detect_changed_translations
python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-detect_changed_translations

# @FIXME: skip detect_changed_source_translations until git diff works again (REV-2737)
check_translations_up_to_date: fake_translations # detect_changed_source_translations

# Validate translations
validate_translations: requirements.tox
tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-validate_translations
python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-validate_translations

# Scan the Django models in all installed apps in this project for restricted field names
check_keywords: requirements.tox
tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-check_keywords
python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-check_keywords

COMMON_CONSTRAINTS_TXT=requirements/common_constraints.txt
.PHONY: $(COMMON_CONSTRAINTS_TXT)
Expand Down Expand Up @@ -173,6 +174,12 @@ upgrade: $(COMMON_CONSTRAINTS_TXT)
docs:
tox -e docs

quality-and-jobs: requirements.js check_translations_up_to_date validate_translations clean_static static quality validate_js check_keywords

test-python: requirements.js clean_static static validate_python

acceptance-python: requirements.js clean_static static acceptance

# Targets in a Makefile which do not produce an output file with the same name as the target name
.PHONY: help requirements migrate serve clean validate_python quality validate_js validate html_coverage e2e \
extract_translations dummy_translations compile_translations fake_translations pull_translations \
Expand Down
6 changes: 5 additions & 1 deletion ecommerce/core/tests/test_models.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@


import json
import sys

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you can remove this from all the files


import ddt
import mock
Expand Down Expand Up @@ -177,7 +178,10 @@ def test_user_details_uses_jwt(self):

# Verify the headers passed to the API were correct.
expected = {'Authorization': 'JWT {}'.format(token), }
self.assertDictContainsSubset(expected, last_request.headers)
if sys.version_info > (3, 9):
self.assertLessEqual(expected.items(), last_request.headers.items())
else:
self.assertDictContainsSubset(expected, last_request.headers)

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
if sys.version_info > (3, 9):
self.assertLessEqual(expected.items(), last_request.headers.items())
else:
self.assertDictContainsSubset(expected, last_request.headers)
self.assertLessEqual(expected.items(), last_request.headers.items())


def test_no_user_details(self):
""" Verify False is returned when there is a connection error. """
Expand Down
15 changes: 13 additions & 2 deletions ecommerce/credit/tests/test_views.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
Tests for the checkout page.
"""


import sys
from datetime import timedelta

import ddt
Expand Down Expand Up @@ -130,7 +130,18 @@ def _assert_success_checkout_page(self, sku=None):

response = self.client.get(self.path)
self.assertEqual(response.status_code, 200)
self.assertDictContainsSubset({'course': self.course}, response.context)
if sys.version_info > (3, 9):
context = {}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A comment here to explain why this is needed would be helpful. Plus, see if this can be improved. There are a bit too many if-elses in the test.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

adding a comment for now.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will also look for the alternative for the if-else.

for i, ctx in enumerate(response.context):
if isinstance(ctx, dict):
context.update(ctx)
elif hasattr(ctx, '__iter__') and not isinstance(ctx, str):
for item in ctx:
if isinstance(item, dict):
context.update(item)
self.assertLessEqual({'course': self.course}.items(), context.items())
else:
self.assertDictContainsSubset({'course': self.course}, response.context)

self.assertContains(
response,
Expand Down
6 changes: 5 additions & 1 deletion ecommerce/enterprise/tests/test_utils.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@


import sys
import uuid

import ddt
Expand Down Expand Up @@ -122,7 +123,10 @@ def test_post_enterprise_customer_user(self, mock_helpers, expected_return):
self.learner.username
)

self.assertDictContainsSubset(expected_return, response)
if sys.version_info > (3, 9):

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same comment as above

self.assertLessEqual(expected_return.items(), response.items())
else:
self.assertDictContainsSubset(expected_return, response)

@responses.activate
def test_ecu_needs_consent(self):
Expand Down
31 changes: 25 additions & 6 deletions ecommerce/extensions/checkout/tests/test_views.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@


import sys
from decimal import Decimal
from urllib import parse

Expand Down Expand Up @@ -338,7 +339,10 @@ def test_get_receipt_for_existing_order(self, mock_learner_data):
}

self.assertEqual(response.status_code, 200)
self.assertDictContainsSubset(context_data, response.context_data)
if sys.version_info > (3, 9):
self.assertEqual(response.context_data, response.context_data | context_data)
else:
self.assertDictContainsSubset(context_data, response.context_data)

@patch('ecommerce.extensions.checkout.views.fetch_enterprise_learner_data')
@responses.activate
Expand Down Expand Up @@ -371,7 +375,10 @@ def test_get_receipt_for_existing_entitlement_order(self, mock_learner_data):
}

self.assertEqual(response.status_code, 200)
self.assertDictContainsSubset(context_data, response.context_data)
if sys.version_info > (3, 9):
self.assertEqual(response.context_data, response.context_data | context_data)
else:
self.assertDictContainsSubset(context_data, response.context_data)

@patch('ecommerce.extensions.checkout.views.fetch_enterprise_learner_data')
@responses.activate
Expand All @@ -387,7 +394,10 @@ def test_get_receipt_for_existing_order_as_staff_user(self, mock_learner_data):
}

self.assertEqual(response.status_code, 200)
self.assertDictContainsSubset(context_data, response.context_data)
if sys.version_info > (3, 9):
self.assertEqual(response.context_data, response.context_data | context_data)
else:
self.assertDictContainsSubset(context_data, response.context_data)

@patch('ecommerce.extensions.checkout.views.fetch_enterprise_learner_data')
@responses.activate
Expand All @@ -400,7 +410,10 @@ def test_get_receipt_for_existing_order_user_not_owner(self, mock_learner_data):
context_data = {'order_history_url': self.site.siteconfiguration.build_lms_url('account/settings')}

self.assertEqual(response.status_code, 404)
self.assertDictContainsSubset(context_data, response.context_data)
if sys.version_info > (3, 9):
self.assertEqual(response.context_data, response.context_data | context_data)
else:
self.assertDictContainsSubset(context_data, response.context_data)

@patch('ecommerce.extensions.checkout.views.fetch_enterprise_learner_data')
@responses.activate
Expand Down Expand Up @@ -456,7 +469,10 @@ def test_dashboard_link_for_course_purchase(self, mock_learner_data):
}

self.assertEqual(response.status_code, 200)
self.assertDictContainsSubset(context_data, response.context_data)
if sys.version_info > (3, 9):
self.assertEqual(response.context_data, response.context_data | context_data)
else:
self.assertDictContainsSubset(context_data, response.context_data)

@patch('ecommerce.extensions.checkout.views.fetch_enterprise_learner_data')
@responses.activate
Expand All @@ -482,7 +498,10 @@ def test_dashboard_link_for_bundle_purchase(self, mock_learner_data):
}

self.assertEqual(response.status_code, 200)
self.assertDictContainsSubset(context_data, response.context_data)
if sys.version_info > (3, 9):
self.assertEqual(response.context_data, response.context_data | context_data)
else:
self.assertDictContainsSubset(context_data, response.context_data)

@patch('ecommerce.extensions.checkout.views.fetch_enterprise_learner_data')
@responses.activate
Expand Down
Loading
Loading