build: add python 3.11 and 3.12 ci checks

.github/workflows/ci.yml

@@ -11,59 +11,71 @@ jobs:
  runs-on: ubuntu-latest
  strategy:
  matrix:
- include:
- - django-env: django32
- testname: quality-and-jobs
- targets: PYTHON_ENV=py38 requirements.js check_translations_up_to_date validate_translations clean_static static quality validate_js check_keywords
- - django-env: django32
- testname: test-python
- targets: PYTHON_ENV=py38 requirements.js clean_static static validate_python
- - django-env: django32
- testname: acceptance-python
- targets: PYTHON_ENV=py38 requirements.js clean_static static acceptance
-
+ python-version: ['py38', 'py311', 'py312']
+ django-env: ['django32']
+ test: ['acceptance-python', 'test-python', 'quality-and-jobs']
  steps:
  - uses: actions/checkout@v2
+ - name: Setup and Format Python Version
+ id: format_python_version
+ shell: bash
+ run: |
+ # Remove 'py' and insert a dot to format the version
+ FORMATTED_VERSION=${{ matrix.python-version }} # e.g., py38
+ FORMATTED_VERSION=${FORMATTED_VERSION/py3/3.} # becomes 3.8
+ # Set environment variables
+ echo "PYTHON_VERSION=$FORMATTED_VERSION" >> $GITHUB_ENV
  - name: Start container
  run: |
- docker-compose -f ./.ci/docker-compose-ci.yml up -d
+ docker compose -f ./.ci/docker-compose-ci.yml up -d
+ docker exec ecommerce_testing bash -c "
+ sudo apt-get update -y &&
+ sudo apt-get install python$PYTHON_VERSION \
+ python$PYTHON_VERSION-dev \
+ python$PYTHON_VERSION-distutils \
+ default-libmysqlclient-dev build-essential pkg-config -y &&
+ curl -sS https://bootstrap.pypa.io/get-pip.py | python$PYTHON_VERSION;"
+ # Need to install pip from source here^ otherwise some packages don't get installed
  - name: Install dependencies
  run: |
  docker exec -t ecommerce_testing bash -c "
  cd /edx/app/ecommerce/ecommerce/ &&
- python3 -m pip install tox
+ python$PYTHON_VERSION -m pip install tox
  "
  - name: Run tests
  run: |
  docker exec -t -e CI=1 ecommerce_testing bash -c "
  cd /edx/app/ecommerce/ecommerce/ &&
  PATH=\$PATH:/edx/app/ecommerce/nodeenvs/ecommerce/bin:/snap/bin
- DJANGO_ENV=${{ matrix.django-env }} make ${{ matrix.targets }}
+ DJANGO_ENV=${{ matrix.django-env }} PYTHON_ENV=${{ matrix.python-version }} PYTHON_VERSION=$PYTHON_VERSION make ${{ matrix.test }}
  "
  - name: Run coverage
- if: matrix.testname == 'test-python'
+ if: matrix.test == 'test-python'
  run: |
  docker exec ecommerce_testing /edx/app/ecommerce/ecommerce/.ci/run_coverage.sh
  - name: Setup Python
- if: matrix.testname == 'test-python'
+ if: matrix.test == 'test-python' && matrix.python-version == 'py38'
  uses: actions/setup-python@v2
  with:
  python-version: "3.8"
  architecture: x64
  - name: Report coverage
- if: matrix.testname == 'test-python'
+ if: matrix.test == 'test-python' && matrix.python-version == 'py38'
  uses: codecov/codecov-action@v3
  with:
  flags: unittests
  fail_ci_if_error: false
 
  docs:
  runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ python-version: ['3.8', '3.11', '3.12']
  steps:
  - uses: actions/checkout@v2
  - uses: actions/setup-python@v2
  with:
- python-version: "3.8"
+ python-version: ${{matrix.python-version}}
  architecture: x64
  - name: Install Dependencies
  run: pip install -r requirements/docs.txt -r requirements/tox.txt

Makefile

@@ -1,7 +1,8 @@
 NODE_BIN=./node_modules/.bin
 DIFF_COVER_BASE_BRANCH=master
-PYTHON_ENV=py38
+PYTHON_ENV_VAR=$(if $(PYTHON_ENV),$(PYTHON_ENV),py312)
 DJANGO_ENV_VAR=$(if $(DJANGO_ENV),$(DJANGO_ENV),django32)
+PYTHON_VERSION_VAR=$(if $(PYTHON_VERSION),$(PYTHON_VERSION),3.12)
 
 help:
  @echo ''
@@ -45,17 +46,17 @@ requirements: requirements.js
  pip3 install -r requirements/dev.txt --exists-action w
 
 requirements.tox:
- pip3 install -U pip==20.0.2
+ pip3 install -U pip
  pip3 install -r requirements/tox.txt --exists-action w
 
 production-requirements: requirements.js
  pip3 install -r requirements.txt --exists-action w
 
 migrate: requirements.tox
- tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-migrate
+ python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-migrate
 
 serve: requirements.tox
- tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-serve
+ python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-serve
 
 clean:
  find . -name '*.pyc' -delete
@@ -65,18 +66,18 @@ clean_static:
  rm -rf assets/* ecommerce/static/build/*
 
 run_check_isort: requirements.tox
- tox -e $(PYTHON_ENV)-check_isort
+ python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-check_isort
 
 run_isort: requirements.tox
- tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-run_isort
+ python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-run_isort
 
 run_pycodestyle: requirements.tox
- tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-pycodestyle
+ python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-pycodestyle
 
 run_pep8: run_pycodestyle
 
 run_pylint: requirements.tox
- tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-pylint
+ python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-pylint
 
 quality: run_check_isort run_pycodestyle run_pylint
 
@@ -86,42 +87,42 @@ validate_js:
  $(NODE_BIN)/gulp lint
 
 validate_python: clean requirements.tox
- tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-tests
+ python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-tests
 
 acceptance: clean requirements.tox
- tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-acceptance
+ python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-acceptance
 
 fast_validate_python: clean requirements.tox
- DISABLE_ACCEPTANCE_TESTS=True tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-tests
+ DISABLE_ACCEPTANCE_TESTS=True python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-tests
 
 validate: validate_python validate_js quality
 
 theme_static: requirements.tox
- tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-theme_static
+ python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-theme_static
 
 static: requirements.js theme_static requirements.tox
  $(NODE_BIN)/r.js -o build.js
- tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-static
+ python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-static
 
 html_coverage: requirements.tox
- tox -e $(PYTHON_ENV)-coverage_html
+ python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-coverage_html
 
 diff_coverage: validate fast_diff_coverage
 
 fast_diff_coverage: requirements.tox
- tox -e $(PYTHON_ENV)-fast_diff_coverage
+ python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-fast_diff_coverage
 
 e2e: requirements.tox
- tox -e $(PYTHON_ENV)-e2e
+ python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-e2e
 
 extract_translations: requirements.tox
- tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-extract_translations
+ python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-extract_translations
 
 dummy_translations: requirements.tox
- tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-dummy_translations
+ python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-dummy_translations
 
 compile_translations: requirements.tox
- tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-compile_translations
+ python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-compile_translations
 
 fake_translations: extract_translations dummy_translations compile_translations
 
@@ -134,18 +135,18 @@ update_translations: pull_translations fake_translations
 
 # extract_translations should be called before this command can detect changes
 detect_changed_source_translations: requirements.tox
- tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-detect_changed_translations
+ python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-detect_changed_translations
 
 # @FIXME: skip detect_changed_source_translations until git diff works again (REV-2737)
 check_translations_up_to_date: fake_translations # detect_changed_source_translations
 
 # Validate translations
 validate_translations: requirements.tox
- tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-validate_translations
+ python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-validate_translations
 
 # Scan the Django models in all installed apps in this project for restricted field names
 check_keywords: requirements.tox
- tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-check_keywords
+ python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-check_keywords
 
 COMMON_CONSTRAINTS_TXT=requirements/common_constraints.txt
 .PHONY: $(COMMON_CONSTRAINTS_TXT)
@@ -173,6 +174,12 @@ upgrade: $(COMMON_CONSTRAINTS_TXT)
 docs:
  tox -e docs
 
+quality-and-jobs: requirements.js check_translations_up_to_date validate_translations clean_static static quality validate_js check_keywords
+
+test-python: requirements.js clean_static static validate_python
+
+acceptance-python: requirements.js clean_static static acceptance
+
 # Targets in a Makefile which do not produce an output file with the same name as the target name
 .PHONY: help requirements migrate serve clean validate_python quality validate_js validate html_coverage e2e \
  extract_translations dummy_translations compile_translations fake_translations pull_translations \

ecommerce/core/tests/test_models.py

@@ -177,7 +177,7 @@ def test_user_details_uses_jwt(self):
 
  # Verify the headers passed to the API were correct.
  expected = {'Authorization': 'JWT {}'.format(token), }
- self.assertDictContainsSubset(expected, last_request.headers)
+ self.assertLessEqual(expected.items(), last_request.headers.items())
 
  def test_no_user_details(self):
  """ Verify False is returned when there is a connection error. """

ecommerce/credit/tests/test_views.py

@@ -2,7 +2,6 @@
 Tests for the checkout page.
 """
 
-
 from datetime import timedelta
 
 import ddt
@@ -130,7 +129,19 @@ def _assert_success_checkout_page(self, sku=None):
 
  response = self.client.get(self.path)
  self.assertEqual(response.status_code, 200)
- self.assertDictContainsSubset({'course': self.course}, response.context)
+
+ # assertDictContainsSubset is deprecated in Python version > 3.9
+ # response.context returns a ContextList object; the below statements will convert it to a dict
+ # assertLessEqual method is used instead of the deprecated assertDictContainsSubset method
+ context = {}
+ for i, ctx in enumerate(response.context):
+ if isinstance(ctx, dict):
+ context.update(ctx)
+ elif hasattr(ctx, '__iter__') and not isinstance(ctx, str):
+ for item in ctx:
+ if isinstance(item, dict):
+ context.update(item)
+ self.assertLessEqual({'course': self.course}.items(), context.items())
 
  self.assertContains(
  response,

ecommerce/enterprise/tests/test_utils.py

@@ -122,7 +122,7 @@ def test_post_enterprise_customer_user(self, mock_helpers, expected_return):
  self.learner.username
  )
 
- self.assertDictContainsSubset(expected_return, response)
+ self.assertLessEqual(expected_return.items(), response.items())
 
  @responses.activate
  def test_ecu_needs_consent(self):

ecommerce/extensions/checkout/tests/test_views.py

@@ -338,7 +338,7 @@ def test_get_receipt_for_existing_order(self, mock_learner_data):
  }
 
  self.assertEqual(response.status_code, 200)
- self.assertDictContainsSubset(context_data, response.context_data)
+ self.assertLessEqual(context_data.items(), response.context_data.items())
 
  @patch('ecommerce.extensions.checkout.views.fetch_enterprise_learner_data')
  @responses.activate
@@ -371,7 +371,7 @@ def test_get_receipt_for_existing_entitlement_order(self, mock_learner_data):
  }
 
  self.assertEqual(response.status_code, 200)
- self.assertDictContainsSubset(context_data, response.context_data)
+ self.assertLessEqual(context_data.items(), response.context_data.items())
 
  @patch('ecommerce.extensions.checkout.views.fetch_enterprise_learner_data')
  @responses.activate
@@ -387,7 +387,7 @@ def test_get_receipt_for_existing_order_as_staff_user(self, mock_learner_data):
  }
 
  self.assertEqual(response.status_code, 200)
- self.assertDictContainsSubset(context_data, response.context_data)
+ self.assertLessEqual(context_data.items(), response.context_data.items())
 
  @patch('ecommerce.extensions.checkout.views.fetch_enterprise_learner_data')
  @responses.activate
@@ -400,7 +400,7 @@ def test_get_receipt_for_existing_order_user_not_owner(self, mock_learner_data):
  context_data = {'order_history_url': self.site.siteconfiguration.build_lms_url('account/settings')}
 
  self.assertEqual(response.status_code, 404)
- self.assertDictContainsSubset(context_data, response.context_data)
+ self.assertLessEqual(context_data.items(), response.context_data.items())
 
  @patch('ecommerce.extensions.checkout.views.fetch_enterprise_learner_data')
  @responses.activate
@@ -456,7 +456,7 @@ def test_dashboard_link_for_course_purchase(self, mock_learner_data):
  }
 
  self.assertEqual(response.status_code, 200)
- self.assertDictContainsSubset(context_data, response.context_data)
+ self.assertLessEqual(context_data.items(), response.context_data.items())
 
  @patch('ecommerce.extensions.checkout.views.fetch_enterprise_learner_data')
  @responses.activate
@@ -482,7 +482,7 @@ def test_dashboard_link_for_bundle_purchase(self, mock_learner_data):
  }
 
  self.assertEqual(response.status_code, 200)
- self.assertDictContainsSubset(context_data, response.context_data)
+ self.assertLessEqual(context_data.items(), response.context_data.items())
 
  @patch('ecommerce.extensions.checkout.views.fetch_enterprise_learner_data')
  @responses.activate

ecommerce/extensions/fulfillment/tests/test_modules.py

@@ -212,7 +212,7 @@ def test_enrollment_module_fulfill(self):
  'X-Forwarded-For': self.user.tracking_context['lms_ip'],
  }
 
- self.assertDictContainsSubset(expected_headers, actual_headers)
+ self.assertLessEqual(expected_headers.items(), actual_headers.items())
  self.assertEqual(expected_body, actual_body)
 
  @responses.activate
@@ -377,7 +377,7 @@ def test_revoke_product(self):
  'X-Forwarded-For': self.user.tracking_context['lms_ip'],
  }
 
- self.assertDictContainsSubset(expected_headers, actual_headers)
+ assert expected_headers.items() <= actual_headers.items()
  self.assertEqual(expected_body, actual_body)
 
  @responses.activate

ecommerce/extensions/payment/tests/processors/test_cybersource.py

@@ -49,7 +49,7 @@ def assert_processor_response_recorded(self, processor_name, transaction_id, res
  expected = {
  'requestID': transaction_id,
  }
- self.assertDictContainsSubset(expected, ppr.response)
+ self.assertLessEqual(expected.items(), ppr.response.items())
  self.assertEqual(ppr.basket, basket)
 
  return ppr.id

ecommerce/extensions/voucher/tests/test_utils.py

@@ -523,7 +523,7 @@ def test_generate_coupon_report_with_deleted_product(self):
 
  __, rows = generate_coupon_report([query_coupon.attr.coupon_vouchers])
  self.assert_report_first_row(rows[0], query_coupon, first_voucher)
- self.assertDictContainsSubset({'Redeemed For Course ID': 'Unknown'}, rows[2])
+ self.assertLessEqual({'Redeemed For Course ID': 'Unknown'}.items(), rows[2].items())
 
  def test_report_for_inactive_coupons(self):
  """ Verify the coupon report show correct status for inactive coupons. """