diff --git a/.github/workflows/tools_tests.yaml b/.github/workflows/tools_tests.yaml index ba597f7000edbe..c2f7d86e694caf 100644 --- a/.github/workflows/tools_tests.yaml +++ b/.github/workflows/tools_tests.yaml @@ -39,6 +39,25 @@ jobs: source selfdrive/test/setup_vsound.sh && \ CI=1 pytest tools/sim/tests/test_metadrive_bridge.py" + test_python311: + name: test python3.11 support + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - name: Installing ubuntu dependencies + run: INSTALL_EXTRA_PACKAGES=no tools/install_ubuntu_dependencies.sh + - name: Installing python + uses: actions/setup-python@v5 + with: + python-version: '3.11.4' + - name: Installing pip + run: pip install pip==24.0 + - name: Installing poetry + run: pip install poetry==1.7.0 + - name: Installing python dependencies + run: poetry install --no-cache --no-root + devcontainer: name: devcontainer runs-on: ubuntu-latest diff --git a/.python-version b/.python-version index 0c7d5f5f5d7616..455808f8e1991a 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.11.4 +3.12.4 diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base index 0789a39c3e8b48..6818195d725606 100644 --- a/Dockerfile.openpilot_base +++ b/Dockerfile.openpilot_base @@ -64,7 +64,7 @@ RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER $USER ENV POETRY_VIRTUALENVS_CREATE=false -ENV PYENV_VERSION=3.11.4 +ENV PYENV_VERSION=3.12.4 ENV PYENV_ROOT="/home/$USER/pyenv" ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH" @@ -76,7 +76,7 @@ RUN cd /tmp && \ rm -rf /tmp/* && \ rm -rf /home/$USER/.cache && \ find /home/$USER/pyenv -type d -name ".git" | xargs rm -rf && \ - rm -rf /home/$USER/pyenv/versions/3.11.4/lib/python3.11/test + rm -rf /home/$USER/pyenv/versions/3.12.4/lib/python3.12/test USER root RUN sudo git config --global --add safe.directory /tmp/openpilot diff --git a/common/api/__init__.py b/common/api/__init__.py index f3a7d83842a5d4..ac231400a47cc7 100644 --- a/common/api/__init__.py +++ b/common/api/__init__.py @@ -1,7 +1,7 @@ import jwt import os import requests -from datetime import datetime, timedelta +from datetime import datetime, timedelta, UTC from openpilot.system.hardware.hw import Paths from openpilot.system.version import get_version @@ -23,7 +23,7 @@ def request(self, method, endpoint, timeout=None, access_token=None, **params): return api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params) def get_token(self, expiry_hours=1): - now = datetime.utcnow() + now = datetime.now(UTC).replace(tzinfo=None) payload = { 'identity': self.dongle_id, 'nbf': now, diff --git a/poetry.lock b/poetry.lock index 2ba149bcc3098f..acbd3209d4daa4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:33726e51a423d7a936f090c33b34d590f74e0694df99b8220e7d0a648b1ac9a6 -size 615418 +oid sha256:08bec056de9bb978a9b50680c699f5a604d7405d105997c0a60be522dcfc7eb5 +size 629836 diff --git a/pyproject.toml b/pyproject.toml index 01885c7ac3d7ea..6e4741ddaf1b64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,7 +88,7 @@ repository = "https://github.com/commaai/openpilot" documentation = "https://docs.comma.ai" [tool.poetry.dependencies] -python = "~3.11" +python = ">=3.11, <3.13" # multiple users sounddevice = "*" # micd + soundd @@ -154,15 +154,14 @@ inputs = "*" Jinja2 = "*" lru-dict = "*" matplotlib = "*" -# No release for this fix https://github.com/metadriverse/metadrive/issues/632. Pinned to this commit until next release -metadrive-simulator = {git = "https://github.com/metadriverse/metadrive.git", rev ="233a3a1698be7038ec3dd050ca10b547b4b3324c", markers = "platform_machine != 'aarch64'" } # no linux/aarch64 wheels for certain dependencies +metadrive-simulator = { git = "https://github.com/commaai/metadrive.git", branch = "python3.12", markers = "platform_machine != 'aarch64'" } # no linux/aarch64 wheels for certain dependencies mpld3 = "*" mypy = "*" myst-parser = "*" natsort = "*" opencv-python-headless = "*" parameterized = "^0.8" -pprofile = "*" +#pprofile = "*" polyline = "*" pre-commit = "*" pyautogui = "*" diff --git a/selfdrive/test/test_updated.py b/selfdrive/test/test_updated.py index f8eae94823420f..ea945d94c29487 100644 --- a/selfdrive/test/test_updated.py +++ b/selfdrive/test/test_updated.py @@ -164,7 +164,7 @@ def _check_update_state(self, update_available): # make sure LastUpdateTime is recent t = self._read_param("LastUpdateTime") last_update_time = datetime.datetime.fromisoformat(t) - td = datetime.datetime.utcnow() - last_update_time + td = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) - last_update_time assert td.total_seconds() < 10 self.params.remove("LastUpdateTime") diff --git a/system/athena/registration.py b/system/athena/registration.py index 6574d9ac202df3..97289e41992ae7 100755 --- a/system/athena/registration.py +++ b/system/athena/registration.py @@ -4,7 +4,7 @@ import jwt from pathlib import Path -from datetime import datetime, timedelta +from datetime import datetime, timedelta, UTC from openpilot.common.api import api_get from openpilot.common.params import Params from openpilot.common.spinner import Spinner @@ -66,7 +66,7 @@ def register(show_spinner=False) -> str | None: start_time = time.monotonic() while True: try: - register_token = jwt.encode({'register': True, 'exp': datetime.utcnow() + timedelta(hours=1)}, private_key, algorithm='RS256') + register_token = jwt.encode({'register': True, 'exp': datetime.now(UTC).replace(tzinfo=None) + timedelta(hours=1)}, private_key, algorithm='RS256') cloudlog.info("getting pilotauth") resp = api_get("v2/pilotauth/", method='POST', timeout=15, imei=imei1, imei2=imei2, serial=serial, public_key=public_key, register_token=register_token) diff --git a/system/manager/manager.py b/system/manager/manager.py index 6d1b8d9c22726c..2a173ac90e739e 100755 --- a/system/manager/manager.py +++ b/system/manager/manager.py @@ -42,7 +42,7 @@ def manager_init() -> None: ("LongitudinalPersonality", str(log.LongitudinalPersonality.standard)), ] if not PC: - default_params.append(("LastUpdateTime", datetime.datetime.utcnow().isoformat().encode('utf8'))) + default_params.append(("LastUpdateTime", datetime.datetime.now(datetime.UTC).replace(tzinfo=None).isoformat().encode('utf8'))) if params.get_bool("RecordFrontLock"): params.put_bool("RecordFront", True) diff --git a/system/qcomgpsd/qcomgpsd.py b/system/qcomgpsd/qcomgpsd.py index 21c7995a779d18..43ddb8993994ca 100755 --- a/system/qcomgpsd/qcomgpsd.py +++ b/system/qcomgpsd/qcomgpsd.py @@ -173,7 +173,7 @@ def setup_quectel(diag: ModemDiag) -> bool: os.remove(ASSIST_DATA_FILE) #at_cmd("AT+QGPSXTRADATA?") if system_time_valid(): - time_str = datetime.datetime.utcnow().strftime("%Y/%m/%d,%H:%M:%S") + time_str = datetime.datetime.now(datetime.UTC).replace(tzinfo=None).strftime("%Y/%m/%d,%H:%M:%S") at_cmd(f"AT+QGPSXTRATIME=0,\"{time_str}\",1,1,1000") at_cmd("AT+QGPSCFG=\"outport\",\"usbnmea\"") diff --git a/system/qcomgpsd/tests/test_qcomgpsd.py b/system/qcomgpsd/tests/test_qcomgpsd.py index ef23737dd7e68c..716bc33ed21b6a 100644 --- a/system/qcomgpsd/tests/test_qcomgpsd.py +++ b/system/qcomgpsd/tests/test_qcomgpsd.py @@ -85,7 +85,7 @@ def check_assistance(self, should_be_loaded): if should_be_loaded: assert valid_duration == "10080" # should be max time injected_time = datetime.datetime.strptime(injected_time_str.replace("\"", ""), "%Y/%m/%d,%H:%M:%S") - assert abs((datetime.datetime.utcnow() - injected_time).total_seconds()) < 60*60*12 + assert abs((datetime.datetime.now(datetime.UTC).replace(tzinfo=None) - injected_time).total_seconds()) < 60*60*12 else: valid_duration, injected_time_str = out.split(",", 1) injected_time_str = injected_time_str.replace('\"', '').replace('\'', '') diff --git a/system/statsd.py b/system/statsd.py index 2b5a7bb3a7721c..5e76b73ae99976 100755 --- a/system/statsd.py +++ b/system/statsd.py @@ -133,7 +133,7 @@ def get_influxdb_line(measurement: str, value: float | dict[str, float], timest # flush when started state changes or after FLUSH_TIME_S if (time.monotonic() > last_flush_time + STATS_FLUSH_TIME_S) or (sm['deviceState'].started != started_prev): result = "" - current_time = datetime.utcnow().replace(tzinfo=UTC) + current_time = datetime.now(UTC) tags['started'] = sm['deviceState'].started for key, value in gauges.items(): diff --git a/system/ubloxd/pigeond.py b/system/ubloxd/pigeond.py index 5711992cfb1aa0..8809689dcddf3c 100755 --- a/system/ubloxd/pigeond.py +++ b/system/ubloxd/pigeond.py @@ -6,7 +6,7 @@ import struct import requests import urllib.parse -from datetime import datetime +from datetime import datetime, UTC from cereal import messaging from openpilot.common.params import Params @@ -196,7 +196,7 @@ def initialize_pigeon(pigeon: TTYPigeon) -> bool: cloudlog.error(f"failed to restore almanac backup, status: {restore_status}") # sending time to ublox - t_now = datetime.utcnow() + t_now = datetime.now(UTC).replace(tzinfo=None) if t_now >= datetime(2021, 6, 1): cloudlog.warning("Sending current time to ublox") diff --git a/system/updated/updated.py b/system/updated/updated.py index d43f439af583c2..005c52bc8bb160 100755 --- a/system/updated/updated.py +++ b/system/updated/updated.py @@ -60,7 +60,7 @@ def sleep(self, t: float) -> None: self.ready_event.wait(timeout=t) def write_time_to_param(params, param) -> None: - t = datetime.datetime.utcnow() + t = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) params.put(param, t.isoformat().encode('utf8')) def read_time_from_param(params, param) -> datetime.datetime | None: @@ -279,7 +279,7 @@ def set_params(self, update_success: bool, failed_count: int, exception: str | N if len(self.branches): self.params.put("UpdaterAvailableBranches", ','.join(self.branches.keys())) - last_update = datetime.datetime.utcnow() + last_update = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) if update_success: write_time_to_param(self.params, "LastUpdateTime") else: @@ -323,7 +323,7 @@ def get_description(basedir: str) -> str: for alert in ("Offroad_UpdateFailed", "Offroad_ConnectivityNeeded", "Offroad_ConnectivityNeededPrompt"): set_offroad_alert(alert, False) - now = datetime.datetime.utcnow() + now = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) dt = now - last_update build_metadata = get_build_metadata() if failed_count > 15 and exception is not None and self.has_internet: @@ -429,7 +429,7 @@ def main() -> None: cloudlog.event("update installed") if not params.get("InstallDate"): - t = datetime.datetime.utcnow().isoformat() + t = datetime.datetime.now(datetime.UTC).replace(tzinfo=None).isoformat() params.put("InstallDate", t.encode('utf8')) updater = Updater() @@ -469,7 +469,7 @@ def main() -> None: # download update last_fetch = read_time_from_param(params, "UpdaterLastFetchTime") - timed_out = last_fetch is None or (datetime.datetime.utcnow() - last_fetch > datetime.timedelta(days=3)) + timed_out = last_fetch is None or (datetime.datetime.now(datetime.UTC).replace(tzinfo=None) - last_fetch > datetime.timedelta(days=3)) user_requested_fetch = wait_helper.user_request == UserRequest.FETCH if params.get_bool("NetworkMetered") and not timed_out and not user_requested_fetch: cloudlog.info("skipping fetch, connection metered") diff --git a/system/webrtc/tests/test_stream_session.py b/system/webrtc/tests/test_stream_session.py index e89e42e440b47a..46d55ecd907a9e 100644 --- a/system/webrtc/tests/test_stream_session.py +++ b/system/webrtc/tests/test_stream_session.py @@ -3,6 +3,7 @@ # for aiortc and its dependencies import warnings warnings.filterwarnings("ignore", category=DeprecationWarning) +warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel from aiortc import RTCDataChannel from aiortc.mediastreams import VIDEO_CLOCK_RATE, VIDEO_TIME_BASE diff --git a/system/webrtc/tests/test_webrtcd.py b/system/webrtc/tests/test_webrtcd.py index d4b659a3aa0e5d..4fa6d8953f7312 100644 --- a/system/webrtc/tests/test_webrtcd.py +++ b/system/webrtc/tests/test_webrtcd.py @@ -4,6 +4,7 @@ # for aiortc and its dependencies import warnings warnings.filterwarnings("ignore", category=DeprecationWarning) +warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel from openpilot.system.webrtc.webrtcd import get_stream diff --git a/system/webrtc/webrtcd.py b/system/webrtc/webrtcd.py index afd346857f0d82..79c5b4888f31be 100755 --- a/system/webrtc/webrtcd.py +++ b/system/webrtc/webrtcd.py @@ -11,6 +11,7 @@ # aiortc and its dependencies have lots of internal warnings :( import warnings warnings.filterwarnings("ignore", category=DeprecationWarning) +warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel import capnp from aiohttp import web diff --git a/tools/lib/azure_container.py b/tools/lib/azure_container.py index f5a3a8bfb197c1..a5d650e98fae4b 100644 --- a/tools/lib/azure_container.py +++ b/tools/lib/azure_container.py @@ -1,5 +1,5 @@ import os -from datetime import datetime, timedelta +from datetime import datetime, timedelta, UTC from functools import lru_cache from pathlib import Path from typing import IO @@ -20,7 +20,7 @@ def get_azure_credential(): @lru_cache def get_container_sas(account_name: str, container_name: str): from azure.storage.blob import BlobServiceClient, ContainerSasPermissions, generate_container_sas - start_time = datetime.utcnow() + start_time = datetime.now(UTC).replace(tzinfo=None) expiry_time = start_time + timedelta(hours=1) blob_service = BlobServiceClient( account_url=f"https://{account_name}.blob.core.windows.net",