From 0a3479e32e3a7b274c396ad329a24738d80fd83a Mon Sep 17 00:00:00 2001 From: Muhammad Soban Javed Date: Wed, 17 Apr 2024 16:52:39 +0500 Subject: [PATCH] build: Add support for Python 3.11 and 3.12 in Dockerfile and CI --- .ci/docker-compose-ci.yml | 7 +++-- .github/workflows/ci.yml | 20 ++++++++++++ Dockerfile | 21 ++++++++++--- .../api/v1/tests/test_views/test_search.py | 31 ++++++++++++++++--- 4 files changed, 68 insertions(+), 11 deletions(-) diff --git a/.ci/docker-compose-ci.yml b/.ci/docker-compose-ci.yml index 4951f9fbbc..db3d61e40c 100644 --- a/.ci/docker-compose-ci.yml +++ b/.ci/docker-compose-ci.yml @@ -29,8 +29,11 @@ services: container_name: memcached discovery: # Uncomment this line to use the official course-discovery base image - image: edxops/discovery:latest - + build: + context: ../. + target: dev + args: + PYTHON_VERSION: "${PYTHON_VERSION}" # Uncomment the next two lines to build from a local configuration repo # build: ../configuration/docker/build/discovery/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42e9ce6327..634f3a0a01 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,20 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Format Python Version + id: format_python_version + shell: bash + run: | + # Remove 'py' and insert a dot to format the version + FORMATTED_VERSION=${{ matrix.python-version }} # e.g., py38 + FORMATTED_VERSION=${FORMATTED_VERSION/py3/3.} # becomes 3.8 + + # Set environment variables + echo "PYTHON_VERSION=$FORMATTED_VERSION" >> $GITHUB_ENV + + # Output formatted version for use in subsequent steps + echo "Using Python Version: $PYTHON_VERSION" + - run: make ci_up - name: run tests env: @@ -42,6 +56,8 @@ jobs: steps: - uses: actions/checkout@v3 - run: make ci_up + env: + PYTHON_VERSION: 3.8 - name: Download all artifacts # Downloads coverage1, coverage2, etc. uses: actions/download-artifact@v2 @@ -54,6 +70,8 @@ jobs: steps: - uses: actions/checkout@v3 - run: make ci_up + env: + PYTHON_VERSION: 3.8 - run: make ci_quality semgrep: @@ -61,5 +79,7 @@ jobs: steps: - uses: actions/checkout@v3 - run: make ci_up + env: + PYTHON_VERSION: 3.8 - name: Run semgrep Django rules run: make ci_semgrep diff --git a/Dockerfile b/Dockerfile index 675a64dfef..dd2f0c89c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,12 @@ FROM ubuntu:focal as app +ARG PYTHON_VERSION=3.8 + ENV DEBIAN_FRONTEND noninteractive # System requirements. -RUN apt update && \ +RUN apt-get update && \ + apt-get install -y software-properties-common && \ + apt-add-repository -y ppa:deadsnakes/ppa && \ apt-get install -qy \ curl \ gettext \ @@ -10,11 +14,14 @@ RUN apt update && \ git \ language-pack-en \ build-essential \ - python3.8-dev \ - python3-virtualenv \ - python3.8-distutils \ + python${PYTHON_VERSION}-dev \ + python${PYTHON_VERSION}-distutils \ libmysqlclient-dev \ libssl-dev \ + # Current version of Pillow (9.5.0) doesn't provide pre-built wheel for python 3.12, + # So this apt package is needed for building Pillow on 3.12, + # and can be removed when version of Pillow is upgraded to 10.5.0+ + libjpeg-dev \ # mysqlclient >= 2.2.0 requires pkg-config. pkg-config \ libcairo2-dev && \ @@ -38,8 +45,12 @@ ENV PATH "${DISCOVERY_VENV_DIR}/bin:${DISCOVERY_NODEENV_DIR}/bin:$PATH" ENV DISCOVERY_CFG "/edx/etc/discovery.yml" ENV DISCOVERY_CODE_DIR "${DISCOVERY_CODE_DIR}" ENV DISCOVERY_APP_DIR "${DISCOVERY_APP_DIR}" +ENV PYTHON_VERSION "${PYTHON_VERSION}" + +RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} +RUN pip install virtualenv -RUN virtualenv -p python3.8 --always-copy ${DISCOVERY_VENV_DIR} +RUN virtualenv -p python${PYTHON_VERSION} --always-copy ${DISCOVERY_VENV_DIR} # No need to activate discovery venv as it is already in path RUN pip install nodeenv diff --git a/course_discovery/apps/api/v1/tests/test_views/test_search.py b/course_discovery/apps/api/v1/tests/test_views/test_search.py index 5e0213a20d..80c86fc0ea 100644 --- a/course_discovery/apps/api/v1/tests/test_views/test_search.py +++ b/course_discovery/apps/api/v1/tests/test_views/test_search.py @@ -1,5 +1,6 @@ import datetime import json +import sys import urllib.parse import uuid @@ -66,7 +67,12 @@ def assert_successful_search(self, path=None, serializer=None): 'next': None, } actual = response_data['objects'] if path == self.faceted_path else response_data - self.assertDictContainsSubset(expected, actual) + if sys.version_info > (3, 9): + # Remove this pylint disable once discovery reaches python 3.11+ + # pylint: disable=unsupported-binary-operation + self.assertEqual(actual, actual | expected) # pragma: no cover + else: + self.assertDictContainsSubset(expected, actual) return course_run, response_data @@ -90,7 +96,13 @@ def assert_response_includes_availability_facets(self, response_data): 'narrow_url': self.build_facet_url({'selected_query_facets': 'availability_upcoming'}) }, } - self.assertDictContainsSubset(expected, response_data['queries']) + + if sys.version_info > (3, 9): + # Remove this pylint disable once discovery reaches python 3.11+ + # pylint: disable=unsupported-binary-operation + self.assertEqual(response_data['queries'], response_data['queries'] | expected) # pragma: no cover + else: + self.assertDictContainsSubset(expected, response_data['queries']) @ddt.data(faceted_path, list_path, detailed_path) def test_authentication(self, path): @@ -117,7 +129,13 @@ def test_faceted_search(self): 'text': course_run.pacing_type, 'count': 1, } - self.assertDictContainsSubset(expected, response_data['fields']['pacing_type'][0]) + actual = response_data['fields']['pacing_type'][0] + if sys.version_info > (3, 9): + # Remove this pylint disable once discovery reaches python 3.11+ + # pylint: disable=unsupported-binary-operation + self.assertEqual(actual, actual | expected) # pragma: no cover + else: + self.assertDictContainsSubset(expected, actual) def test_invalid_query_facet(self): """ Verify the endpoint returns HTTP 400 if an invalid facet is requested. """ @@ -202,7 +220,12 @@ def test_exclude_unavailable_program_types(self, path, serializer, result_locati self.serialize_course_run_search(course_run, serializer=serializer) ] } - self.assertDictContainsSubset(expected, response_data) + if sys.version_info > (3, 9): + # Remove this pylint disable once discovery reaches python 3.11+ + # pylint: disable=unsupported-binary-operation + self.assertEqual(response_data, response_data | expected) # pragma: no cover + else: + self.assertDictContainsSubset(expected, response_data) # Check that the program is indeed the active one. for key in result_location_keys: