Upgrade to Python 3.9 (#1844)

* Upgrade to Python 3.9

* fix

.github/workflows/ci.yaml

@@ -30,12 +30,12 @@ jobs:
     - name: Install python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.8.10
+        python-version: 3.9.9
     - name: Cache packages
       uses: actions/[email protected]
       with:
         path: ~/.cache/pip
-        key: ${{ runner.os }}-py-3.8.10-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('setup.cfg') }}
+        key: ${{ runner.os }}-py-3.9.9-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('setup.cfg') }}
     - name: Install dependencies
       run: make setup
     - name: Lint
@@ -113,7 +113,7 @@ jobs:
     - name: Install python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.8.10
+        python-version: 3.9.9
     - name: Install Helm
       uses: azure/setup-helm@v1
       with:

.pre-commit-config.yaml

@@ -38,11 +38,15 @@ repos:
     files: |
       docs/spelling_wordlist.txt|
       .gitignore
+- repo: https://github.com/sondrelg/pep585-upgrade
+  rev: 'v1.0.1'
+  hooks:
+  - id: upgrade-type-hints
 - repo: https://github.com/asottile/pyupgrade
   rev: 'v2.29.1'
   hooks:
   - id: pyupgrade
-    args: ['--py36-plus']
+    args: ['--py39-plus']
 - repo: https://gitlab.com/pycqa/flake8
   rev: '3.9.2'
   hooks:

Dockerfile

@@ -1,7 +1,4 @@
-ARG PYTHON_VERSION=3.8.12
-ARG PYTHON_BASE=buster
-
-FROM python:${PYTHON_VERSION} AS installer
+FROM python:3.9.9-slim-bullseye AS installer
 
 ENV PATH=/root/.local/bin:$PATH
 
@@ -12,7 +9,7 @@ COPY dist /tmp/dist
 RUN ls /tmp/dist
 RUN pip install --user --find-links /tmp/dist platform-api
 
-FROM python:${PYTHON_VERSION}-${PYTHON_BASE} AS service
+FROM python:3.9.9-slim-bullseye AS service
 
 LABEL org.opencontainers.image.source = "https://github.com/neuro-inc/platform-api"
 

Makefile

@@ -52,9 +52,7 @@ docker_build:
 	rm -rf build dist
 	pip install -U build
 	python -m build
-	docker build \
-		--build-arg PYTHON_BASE=slim-buster \
-		-t $(IMAGE_NAME):latest .
+	docker build -t $(IMAGE_NAME):latest .
 
 docker_push:
 	docker tag $(IMAGE_NAME):latest $(IMAGE_REPO):$(IMAGE_TAG)

platform_api/api.py

@@ -1,11 +1,12 @@
 import asyncio
 import logging
+from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
 from contextlib import AsyncExitStack
-from typing import Any, AsyncIterator, Awaitable, Callable, Dict, List, Sequence
+from importlib.metadata import version
+from typing import Any
 
 import aiohttp.web
 import aiohttp_cors
-import pkg_resources
 from aiohttp.web import HTTPUnauthorized
 from aiohttp.web_urldispatcher import AbstractRoute
 from aiohttp_security import check_permission
@@ -62,7 +63,7 @@
 
 
 class ApiHandler:
-    def register(self, app: aiohttp.web.Application) -> List[AbstractRoute]:
+    def register(self, app: aiohttp.web.Application) -> list[AbstractRoute]:
         return app.add_routes((aiohttp.web.get("/ping", self.handle_ping),))
 
     @notrace
@@ -104,7 +105,7 @@ async def handle_clusters_sync(
         return aiohttp.web.Response(text="OK")
 
     async def handle_config(self, request: aiohttp.web.Request) -> aiohttp.web.Response:
-        data: Dict[str, Any] = {}
+        data: dict[str, Any] = {}
 
         try:
             user = await authorized_user(request)
@@ -147,7 +148,7 @@ async def handle_config(self, request: aiohttp.web.Request) -> aiohttp.web.Respo
 
     def _convert_cluster_config_to_payload(
         self, user_cluster_config: UserClusterConfig
-    ) -> Dict[str, Any]:
+    ) -> dict[str, Any]:
         cluster_config = user_cluster_config.config
         orgs = user_cluster_config.orgs
         presets = [
@@ -169,8 +170,8 @@ def _convert_cluster_config_to_payload(
             "orgs": orgs,
         }
 
-    def _convert_preset_to_payload(self, preset: Preset) -> Dict[str, Any]:
-        payload: Dict[str, Any] = {
+    def _convert_preset_to_payload(self, preset: Preset) -> dict[str, Any]:
+        payload: dict[str, Any] = {
             "name": preset.name,
             "credits_per_hour": str(preset.credits_per_hour),
             "cpu": preset.cpu,
@@ -243,7 +244,7 @@ async def create_jobs_app(config: Config) -> aiohttp.web.Application:
     return jobs_app
 
 
-package_version = pkg_resources.get_distribution("platform-api").version
+package_version = version(__package__)
 
 
 async def add_version_to_header(
@@ -252,7 +253,7 @@ async def add_version_to_header(
     response.headers["X-Service-Version"] = f"platform-api/{package_version}"
 
 
-def make_tracing_trace_configs(config: Config) -> List[aiohttp.TraceConfig]:
+def make_tracing_trace_configs(config: Config) -> list[aiohttp.TraceConfig]:
     trace_configs = []
 
     if config.zipkin:

platform_api/cluster.py

@@ -1,8 +1,9 @@
 import asyncio
 import logging
 from abc import ABC, abstractmethod
+from collections.abc import AsyncIterator, Sequence
 from contextlib import asynccontextmanager
-from typing import Any, AsyncIterator, Callable, Dict, List, Optional, Sequence
+from typing import Any, Callable, Optional
 
 from aiorwlock import RWLock
 
@@ -216,10 +217,10 @@ class ClusterConfigRegistry:
     def __init__(
         self,
     ) -> None:
-        self._records: Dict[str, ClusterConfig] = {}
+        self._records: dict[str, ClusterConfig] = {}
 
     @property
-    def cluster_names(self) -> List[str]:
+    def cluster_names(self) -> list[str]:
         return list(self._records)
 
     def get(self, name: str) -> ClusterConfig:

platform_api/cluster_config.py

@@ -1,7 +1,8 @@
+from collections.abc import Sequence
 from dataclasses import dataclass, field
 from enum import Enum
 from pathlib import PurePath
-from typing import Optional, Sequence
+from typing import Optional
 
 from yarl import URL
 

platform_api/cluster_config_factory.py

@@ -1,6 +1,7 @@
 import logging
+from collections.abc import Sequence
 from decimal import Decimal
-from typing import Any, Dict, List, Optional, Sequence
+from typing import Any, Optional
 
 import trafaret as t
 from yarl import URL
@@ -14,12 +15,12 @@
 
 class ClusterConfigFactory:
     def create_cluster_configs(
-        self, payload: Sequence[Dict[str, Any]]
+        self, payload: Sequence[dict[str, Any]]
     ) -> Sequence[ClusterConfig]:
         configs = (self.create_cluster_config(p) for p in payload)
         return [c for c in configs if c]
 
-    def create_cluster_config(self, payload: Dict[str, Any]) -> Optional[ClusterConfig]:
+    def create_cluster_config(self, payload: dict[str, Any]) -> Optional[ClusterConfig]:
         try:
             _cluster_config_validator.check(payload)
             return ClusterConfig(
@@ -31,7 +32,7 @@ def create_cluster_config(self, payload: Dict[str, Any]) -> Optional[ClusterConf
             logging.warning(f"failed to parse cluster config: {err}")
             return None
 
-    def _create_ingress_config(self, payload: Dict[str, Any]) -> IngressConfig:
+    def _create_ingress_config(self, payload: dict[str, Any]) -> IngressConfig:
         return IngressConfig(
             registry_url=URL(payload["registry"]["url"]),
             storage_url=URL(payload["storage"]["url"]),
@@ -43,7 +44,7 @@ def _create_ingress_config(self, payload: Dict[str, Any]) -> IngressConfig:
             buckets_url=URL(payload["buckets"]["url"]),
         )
 
-    def _create_presets(self, payload: Dict[str, Any]) -> List[Preset]:
+    def _create_presets(self, payload: dict[str, Any]) -> list[Preset]:
         result = []
         for preset in payload.get("resource_presets", []):
             result.append(
@@ -64,7 +65,7 @@ def _create_presets(self, payload: Dict[str, Any]) -> List[Preset]:
         return result
 
     def _create_orchestrator_config(
-        self, payload: Dict[str, Any]
+        self, payload: dict[str, Any]
     ) -> OrchestratorConfig:
         orchestrator = payload["orchestrator"]
         presets = self._create_presets(orchestrator)
@@ -93,7 +94,7 @@ def _create_orchestrator_config(
         )
 
     def _create_tpu_preset(
-        self, payload: Optional[Dict[str, Any]]
+        self, payload: Optional[dict[str, Any]]
     ) -> Optional[TPUPreset]:
         if not payload:
             return None
@@ -102,7 +103,7 @@ def _create_tpu_preset(
             type=payload["type"], software_version=payload["software_version"]
         )
 
-    def _create_resource_pool_type(self, payload: Dict[str, Any]) -> ResourcePoolType:
+    def _create_resource_pool_type(self, payload: dict[str, Any]) -> ResourcePoolType:
         cpu = payload.get("cpu")
         memory_mb = payload.get("memory_mb")
         return ResourcePoolType(
@@ -121,7 +122,7 @@ def _create_resource_pool_type(self, payload: Dict[str, Any]) -> ResourcePoolTyp
         )
 
     def _create_tpu_resource(
-        self, payload: Optional[Dict[str, Any]]
+        self, payload: Optional[dict[str, Any]]
     ) -> Optional[TPUResource]:
         if not payload:
             return None

platform_api/config.py

@@ -1,7 +1,8 @@
+from collections.abc import Sequence
 from dataclasses import dataclass, field
 from datetime import timedelta
 from decimal import Decimal
-from typing import Optional, Sequence
+from typing import Optional
 
 from alembic.config import Config as AlembicConfig
 from yarl import URL

platform_api/config_client.py

@@ -1,5 +1,6 @@
+from collections.abc import AsyncIterator, Sequence
 from contextlib import asynccontextmanager
-from typing import Any, AsyncIterator, Optional, Sequence
+from typing import Any, Optional
 
 import aiohttp
 from multidict import CIMultiDict

platform_api/config_factory.py

@@ -1,8 +1,9 @@
 import os
 import pathlib
+from collections.abc import Sequence
 from decimal import Decimal
 from pathlib import PurePath
-from typing import Dict, List, Optional, Sequence
+from typing import Optional
 
 from alembic.config import Config as AlembicConfig
 from yarl import URL
@@ -29,7 +30,7 @@
 
 
 class EnvironConfigFactory:
-    def __init__(self, environ: Optional[Dict[str, str]] = None):
+    def __init__(self, environ: Optional[dict[str, str]] = None):
         self._environ = environ or os.environ
 
     def _get_bool(self, name: str, default: bool = False) -> bool:
@@ -331,7 +332,7 @@ def create_registry(self) -> RegistryConfig:
         )
 
     def create_storages(self) -> Sequence[StorageConfig]:
-        result: List[StorageConfig] = []
+        result: list[StorageConfig] = []
         i = 0
 
         while True:

platform_api/handlers/job_request_builder.py

@@ -1,5 +1,5 @@
 from pathlib import PurePath
-from typing import Any, Dict
+from typing import Any
 
 from platform_api.orchestrator.job_request import (
     Container,
@@ -13,7 +13,7 @@
 )
 
 
-def create_container_from_payload(payload: Dict[str, Any]) -> Container:
+def create_container_from_payload(payload: dict[str, Any]) -> Container:
     if "container" in payload:
         # Deprecated. Use flat structure
         payload = payload["container"]
@@ -62,7 +62,7 @@ def create_container_from_payload(payload: Dict[str, Any]) -> Container:
     )
 
 
-def create_resources_from_payload(payload: Dict[str, Any]) -> ContainerResources:
+def create_resources_from_payload(payload: dict[str, Any]) -> ContainerResources:
     tpu = None
     if "tpu" in payload:
         tpu = create_tpu_resource_from_payload(payload["tpu"])
@@ -76,13 +76,13 @@ def create_resources_from_payload(payload: Dict[str, Any]) -> ContainerResources
     )
 
 
-def create_tpu_resource_from_payload(payload: Dict[str, Any]) -> ContainerTPUResource:
+def create_tpu_resource_from_payload(payload: dict[str, Any]) -> ContainerTPUResource:
     return ContainerTPUResource(
         type=payload["type"], software_version=payload["software_version"]
     )
 
 
-def create_volume_from_payload(payload: Dict[str, Any]) -> ContainerVolume:
+def create_volume_from_payload(payload: dict[str, Any]) -> ContainerVolume:
     dst_path = PurePath(payload["dst_path"])
     return ContainerVolume.create(
         payload["src_storage_uri"],
@@ -91,13 +91,13 @@ def create_volume_from_payload(payload: Dict[str, Any]) -> ContainerVolume:
     )
 
 
-def create_secret_volume_from_payload(payload: Dict[str, Any]) -> SecretContainerVolume:
+def create_secret_volume_from_payload(payload: dict[str, Any]) -> SecretContainerVolume:
     return SecretContainerVolume.create(
         uri=payload["src_secret_uri"], dst_path=PurePath(payload["dst_path"])
     )
 
 
-def create_disk_volume_from_payload(payload: Dict[str, Any]) -> DiskContainerVolume:
+def create_disk_volume_from_payload(payload: dict[str, Any]) -> DiskContainerVolume:
     return DiskContainerVolume.create(
         uri=payload["src_disk_uri"],
         dst_path=PurePath(payload["dst_path"]),