build: Add support for Python 3.11 and 3.12 in Dockerfile and CI

.ci/docker-compose-ci.yml

@@ -29,8 +29,11 @@ services:
     container_name: memcached
   discovery:
     # Uncomment this line to use the official course-discovery base image
-    image: edxops/discovery:latest
-
+    build:
+      context: ../.
+      target: dev
+      args:
+        PYTHON_VERSION: "${PYTHON_VERSION}"
     # Uncomment the next two lines to build from a local configuration repo
     # build: ../configuration/docker/build/discovery/
 

.github/workflows/ci.yml

@@ -18,6 +18,20 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+      - name: Format Python Version
+        id: format_python_version
+        shell: bash
+        run: |
+          # Remove 'py' and insert a dot to format the version
+          FORMATTED_VERSION=${{ matrix.python-version }}  # e.g., py38
+          FORMATTED_VERSION=${FORMATTED_VERSION/py3/3.}    # becomes 3.8
+          
+          # Set environment variables
+          echo "PYTHON_VERSION=$FORMATTED_VERSION" >> $GITHUB_ENV
+          
+          # Output formatted version for use in subsequent steps
+          echo "Using Python Version: $PYTHON_VERSION"
+
       - run: make ci_up
       - name: run tests
         env:
@@ -42,6 +56,8 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - run: make ci_up
+        env:
+          PYTHON_VERSION: 3.8
       - name: Download all artifacts
         # Downloads coverage1, coverage2, etc.
         uses: actions/download-artifact@v2
@@ -54,12 +70,16 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - run: make ci_up
+        env:
+          PYTHON_VERSION: 3.8
       - run: make ci_quality
 
   semgrep:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - run: make ci_up
+        env:
+          PYTHON_VERSION: 3.8
       - name: Run semgrep Django rules
         run: make ci_semgrep

Dockerfile

@@ -1,20 +1,27 @@
 FROM ubuntu:focal as app
 
+ARG PYTHON_VERSION=3.8
+
 ENV DEBIAN_FRONTEND noninteractive
 # System requirements.
-RUN apt update && \
+RUN apt-get update && \
+  apt-get install -y software-properties-common && \
+  apt-add-repository -y ppa:deadsnakes/ppa && \
   apt-get install -qy \
   curl \
   gettext \
   # required by bower installer
   git \
   language-pack-en \
   build-essential \
-  python3.8-dev \
-  python3-virtualenv \
-  python3.8-distutils \
+  python${PYTHON_VERSION}-dev \
+  python${PYTHON_VERSION}-distutils \
   libmysqlclient-dev \
   libssl-dev \
+  # Current version of Pillow (9.5.0) doesn't provide pre-built wheel for python 3.12,
+  # So this apt package is needed for building Pillow on 3.12,
+  # and can be removed when version of Pillow is upgraded to 10.5.0+
+  libjpeg-dev \
   # mysqlclient >= 2.2.0 requires pkg-config.
   pkg-config \
   libcairo2-dev && \
@@ -38,8 +45,12 @@ ENV PATH "${DISCOVERY_VENV_DIR}/bin:${DISCOVERY_NODEENV_DIR}/bin:$PATH"
 ENV DISCOVERY_CFG "/edx/etc/discovery.yml"
 ENV DISCOVERY_CODE_DIR "${DISCOVERY_CODE_DIR}"
 ENV DISCOVERY_APP_DIR "${DISCOVERY_APP_DIR}"
+ENV PYTHON_VERSION "${PYTHON_VERSION}"
+
+RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION}
+RUN pip install virtualenv
 
-RUN virtualenv -p python3.8 --always-copy ${DISCOVERY_VENV_DIR}
+RUN virtualenv -p python${PYTHON_VERSION} --always-copy ${DISCOVERY_VENV_DIR}
 
 # No need to activate discovery venv as it is already in path
 RUN pip install nodeenv

course_discovery/apps/api/v1/tests/test_views/test_search.py

@@ -1,5 +1,6 @@
 import datetime
 import json
+import sys
 import urllib.parse
 import uuid
 
@@ -66,7 +67,12 @@ def assert_successful_search(self, path=None, serializer=None):
             'next': None,
         }
         actual = response_data['objects'] if path == self.faceted_path else response_data
-        self.assertDictContainsSubset(expected, actual)
+        if sys.version_info > (3, 9):
+            # Remove this pylint disable once discovery reaches python 3.11+
+            # pylint: disable=unsupported-binary-operation
+            self.assertEqual(actual, actual | expected)  # pragma: no cover
+        else:
+            self.assertDictContainsSubset(expected, actual)
 
         return course_run, response_data
 
@@ -90,7 +96,13 @@ def assert_response_includes_availability_facets(self, response_data):
                 'narrow_url': self.build_facet_url({'selected_query_facets': 'availability_upcoming'})
             },
         }
-        self.assertDictContainsSubset(expected, response_data['queries'])
+
+        if sys.version_info > (3, 9):
+            # Remove this pylint disable once discovery reaches python 3.11+
+            # pylint: disable=unsupported-binary-operation
+            self.assertEqual(response_data['queries'], response_data['queries'] | expected)  # pragma: no cover
+        else:
+            self.assertDictContainsSubset(expected, response_data['queries'])
 
     @ddt.data(faceted_path, list_path, detailed_path)
     def test_authentication(self, path):
@@ -117,7 +129,13 @@ def test_faceted_search(self):
             'text': course_run.pacing_type,
             'count': 1,
         }
-        self.assertDictContainsSubset(expected, response_data['fields']['pacing_type'][0])
+        actual = response_data['fields']['pacing_type'][0]
+        if sys.version_info > (3, 9):
+            # Remove this pylint disable once discovery reaches python 3.11+
+            # pylint: disable=unsupported-binary-operation
+            self.assertEqual(actual, actual | expected)  # pragma: no cover
+        else:
+            self.assertDictContainsSubset(expected, actual)
 
     def test_invalid_query_facet(self):
         """ Verify the endpoint returns HTTP 400 if an invalid facet is requested. """
@@ -202,7 +220,12 @@ def test_exclude_unavailable_program_types(self, path, serializer, result_locati
                 self.serialize_course_run_search(course_run, serializer=serializer)
             ]
         }
-        self.assertDictContainsSubset(expected, response_data)
+        if sys.version_info > (3, 9):
+            # Remove this pylint disable once discovery reaches python 3.11+
+            # pylint: disable=unsupported-binary-operation
+            self.assertEqual(response_data, response_data | expected)  # pragma: no cover
+        else:
+            self.assertDictContainsSubset(expected, response_data)
 
         # Check that the program is indeed the active one.
         for key in result_location_keys: