add resource_presets management commands

CHANGELOG.D/1863.feature

@@ -0,0 +1,2 @@
+Added cluster resource presets management commands.
+Added `Preemptible Node` column in resource presets format in `neuro config show`.

CLI.md

@@ -12,6 +12,8 @@
 		* [neuro admin remove-cluster-user](#neuro-admin-remove-cluster-user)
 		* [neuro admin set-user-quota](#neuro-admin-set-user-quota)
 		* [neuro admin add-user-quota](#neuro-admin-add-user-quota)
+		* [neuro admin update-resource-preset](#neuro-admin-update-resource-preset)
+		* [neuro admin remove-resource-preset](#neuro-admin-remove-resource-preset)
 	* [neuro job](#neuro-job)
 		* [neuro job run](#neuro-job-run)
 		* [neuro job ls](#neuro-job-ls)
@@ -202,6 +204,8 @@ Name | Description|
 | _[neuro admin remove\-cluster-user](#neuro-admin-remove-cluster-user)_| Remove user access from the cluster |
 | _[neuro admin set\-user-quota](#neuro-admin-set-user-quota)_| Set user quota to given values |
 | _[neuro admin add\-user-quota](#neuro-admin-add-user-quota)_| Add given values to user quota |
+| _[neuro admin update\-resource-preset](#neuro-admin-update-resource-preset)_| Add/update resource preset |
+| _[neuro admin remove\-resource-preset](#neuro-admin-remove-resource-preset)_| Remove resource preset |
 
 
 
@@ -384,6 +388,52 @@ Name | Description|
 
 
 
+### neuro admin update-resource-preset
+
+Add/update resource preset
+
+**Usage:**
+
+```bash
+neuro admin update-resource-preset [OPTIONS] CLUSTER_NAME PRESET_NAME
+```
+
+**Options:**
+
+Name | Description|
+|----|------------|
+|_--help_|Show this message and exit.|
+|_\-c, --cpu NUMBER_|Number of CPUs  \[default: 0.1]|
+|_\-g, --gpu NUMBER_|Number of GPUs|
+|_\--gpu-model MODEL_|GPU model|
+|_\-m, --memory AMOUNT_|Memory amount  \[default: 1G]|
+|_\-p, --preemptible / -P, --non-preemptible_|Job preemptability support  \[default: False]|
+|_\--preemptible-node / --non-preemptible-node_|Use a lower-cost preemptible instance  \[default: False]|
+|_\--tpu-sw-version VERSION_|TPU software version|
+|_\--tpu-type TYPE_|TPU type|
+
+
+
+
+### neuro admin remove-resource-preset
+
+Remove resource preset
+
+**Usage:**
+
+```bash
+neuro admin remove-resource-preset [OPTIONS] CLUSTER_NAME PRESET_NAME
+```
+
+**Options:**
+
+Name | Description|
+|----|------------|
+|_--help_|Show this message and exit.|
+
+
+
+
 ## neuro job
 
 Job operations.

neuromation/api/admin.py

@@ -4,6 +4,7 @@
 
 from neuromation.api.config import Config
 from neuromation.api.core import _Core
+from neuromation.api.server_cfg import Preset
 from neuromation.api.utils import NoPublicConstructor
 
 
@@ -99,6 +100,17 @@ async def add_cluster(self, name: str, config: Dict[str, Any]) -> None:
         async with self._core.request("PUT", url, auth=auth, json=config) as resp:
             resp
 
+    async def update_cluster_resource_presets(
+        self, name: str, presets: Mapping[str, Preset]
+    ) -> None:
+        url = self._config.api_url / "clusters" / name / "orchestrator/resource_presets"
+        auth = await self._config._api_auth()
+        payload = [
+            _serialize_resource_preset(name, preset) for name, preset in presets.items()
+        ]
+        async with self._core.request("PUT", url, auth=auth, json=payload):
+            pass
+
     async def list_cluster_users(
         self, cluster_name: Optional[str] = None
     ) -> List[_ClusterUser]:
@@ -264,3 +276,22 @@ def _node_pool_from_api(payload: Dict[str, Any]) -> _NodePool:
 
 def _storage_from_api(payload: Dict[str, Any]) -> _Storage:
     return _Storage(description=payload["description"])
+
+
+def _serialize_resource_preset(name: str, preset: Preset) -> Dict[str, Any]:
+    result: Dict[str, Any] = {
+        "name": name,
+        "cpu": preset.cpu,
+        "memory_mb": preset.memory_mb,
+        "is_preemptible": preset.is_preemptible,
+        "is_preemptible_node_required": preset.is_preemptible_node_required,
+    }
+    if preset.gpu:
+        result["gpu"] = preset.gpu
+        result["gpu_model"] = preset.gpu_model
+    if preset.tpu_type and preset.tpu_software_version:
+        result["tpu"] = {
+            "type": preset.tpu_type,
+            "software_version": preset.tpu_software_version,
+        }
+    return result

neuromation/api/config.py

@@ -399,6 +399,9 @@ def _deserialize_resource_preset(payload: Dict[str, Any]) -> Tuple[str, Preset]:
             tpu_type=payload.get("tpu_type", None),
             tpu_software_version=payload.get("tpu_software_version", None),
             is_preemptible=payload.get("is_preemptible", False),
+            is_preemptible_node_required=payload.get(
+                "is_preemptible_node_required", False
+            ),
         ),
     )
 
@@ -507,6 +510,7 @@ def _serialize_resource_preset(name: str, preset: Preset) -> Dict[str, Any]:
         "tpu_type": preset.tpu_type,
         "tpu_software_version": preset.tpu_software_version,
         "is_preemptible": preset.is_preemptible,
+        "is_preemptible_node_required": preset.is_preemptible_node_required,
     }
 
 

neuromation/api/server_cfg.py

@@ -13,6 +13,7 @@ class Preset:
     cpu: float
     memory_mb: int
     is_preemptible: bool = False
+    is_preemptible_node_required: bool = False
     gpu: Optional[int] = None
     gpu_model: Optional[str] = None
     tpu_type: Optional[str] = None

neuromation/cli/admin.py

@@ -9,7 +9,10 @@
 from prompt_toolkit import PromptSession
 
 from neuromation.api.admin import _ClusterUserRoleType
+from neuromation.api.server_cfg import Preset
 
+from .click_types import MEGABYTE
+from .defaults import JOB_CPU_NUMBER, JOB_MEMORY_AMOUNT
 from .formatters.admin import ClustersFormatter, ClusterUserFormatter
 from .formatters.config import QuotaFormatter
 from .root import Root
@@ -403,6 +406,116 @@ async def add_user_quota(
     root.print(fmt(user_with_quota.quota))
 
 
+@command()
+@argument("cluster_name")
+@argument("preset_name")
+@option(
+    "-c",
+    "--cpu",
+    metavar="NUMBER",
+    type=float,
+    help="Number of CPUs",
+    default=JOB_CPU_NUMBER,
+    show_default=True,
+)
+@option(
+    "-m",
+    "--memory",
+    metavar="AMOUNT",
+    type=MEGABYTE,
+    help="Memory amount",
+    default=JOB_MEMORY_AMOUNT,
+    show_default=True,
+)
+@option(
+    "-g",
+    "--gpu",
+    metavar="NUMBER",
+    type=int,
+    help="Number of GPUs",
+)
+@option(
+    "--gpu-model",
+    metavar="MODEL",
+    help="GPU model",
+)
+@option("--tpu-type", metavar="TYPE", type=str, help="TPU type")
+@option(
+    "tpu_software_version",
+    "--tpu-sw-version",
+    metavar="VERSION",
+    type=str,
+    help="TPU software version",
+)
+@option(
+    "--preemptible/--non-preemptible",
+    "-p/-P",
+    help="Job preemptability support",
+    default=False,
+    show_default=True,
+)
+@option(
+    "--preemptible-node/--non-preemptible-node",
+    help="Use a lower-cost preemptible instance",
+    default=False,
+    show_default=True,
+)
+async def update_resource_preset(
+    root: Root,
+    cluster_name: str,
+    preset_name: str,
+    cpu: float,
+    memory: int,
+    gpu: Optional[int],
+    gpu_model: Optional[str],
+    tpu_type: Optional[str],
+    tpu_software_version: Optional[str],
+    preemptible: bool,
+    preemptible_node: bool,
+) -> None:
+    """
+    Add/update resource preset
+    """
+    presets = dict(root.client.presets)
+    presets[preset_name] = Preset(
+        cpu=cpu,
+        memory_mb=memory,
+        gpu=gpu,
+        gpu_model=gpu_model,
+        tpu_type=tpu_type,
+        tpu_software_version=tpu_software_version,
+        is_preemptible=preemptible,
+        is_preemptible_node_required=preemptible_node,
+    )
+    await root.client._admin.update_cluster_resource_presets(cluster_name, presets)
+    if not root.quiet:
+        root.print(
+            f"Updated resource preset {click.style(preset_name, bold=True)} "
+            f"in cluster {click.style(cluster_name, bold=True)}"
+        )
+
+
+@command()
+@argument("cluster_name")
+@argument("preset_name")
+async def remove_resource_preset(
+    root: Root, cluster_name: str, preset_name: str
+) -> None:
+    """
+    Remove resource preset
+    """
+    presets = dict(root.client.presets)
+    if preset_name not in presets:
+        raise ValueError(f"Preset '{preset_name}' not found")
+    del presets[preset_name]
+    await root.client._admin.update_cluster_resource_presets(cluster_name, presets)
+    if not root.quiet:
+        root.print(
+            f"Removed resource preset {click.style(preset_name, bold=True)} "
+            f"from cluster {click.style(cluster_name, bold=True)}"
+        )
+
+
 admin.add_command(get_clusters)
 admin.add_command(generate_cluster_config)
 admin.add_command(add_cluster)
@@ -414,3 +527,6 @@ async def add_user_quota(
 
 admin.add_command(set_user_quota)
 admin.add_command(add_user_quota)
+
+admin.add_command(update_resource_preset)
+admin.add_command(remove_resource_preset)

neuromation/cli/formatters/config.py

@@ -132,6 +132,7 @@ def _format_presets(
     table.add_column("#CPU", justify="right")
     table.add_column("Memory", justify="right")
     table.add_column("Preemptible", justify="center")
+    table.add_column("Preemptible Node", justify="center")
     table.add_column("GPU", justify="left")
     if available_jobs_counts:
         table.add_column("Jobs Avail", justify="right")
@@ -147,6 +148,7 @@ def _format_presets(
             str(preset.cpu),
             format_size(preset.memory_mb * 1024 ** 2),
             "√" if preset.is_preemptible else "×",
+            "√" if preset.is_preemptible_node_required else "×",
             gpu,
         ]
         if has_tpu:

tests/api/test_admin.py

@@ -13,6 +13,7 @@
     _NodePool,
     _Storage,
 )
+from neuromation.api.server_cfg import Preset
 from tests import _TestServerFactory
 
 
@@ -504,3 +505,74 @@ async def handle_cloud_providers(
     async with make_client(srv.make_url("/api/v1")) as client:
         result = await client._admin.get_cloud_provider_options("aws")
         assert result == sample_response
+
+
+async def test_update_cluster_resource_presets(
+    aiohttp_server: _TestServerFactory, make_client: _MakeClient
+) -> None:
+    async def update_cluster_resource_presets(
+        request: web.Request,
+    ) -> web.StreamResponse:
+        assert request.match_info["cluster_name"] == "my_cluster"
+        assert sorted(await request.json(), key=lambda x: x["name"]) == [
+            {
+                "name": "cpu-micro",
+                "cpu": 0.1,
+                "memory_mb": 100,
+                "is_preemptible": False,
+                "is_preemptible_node_required": False,
+            },
+            {
+                "name": "cpu-micro-p",
+                "cpu": 0.1,
+                "memory_mb": 100,
+                "is_preemptible": True,
+                "is_preemptible_node_required": True,
+            },
+            {
+                "name": "gpu-micro",
+                "cpu": 0.2,
+                "memory_mb": 200,
+                "gpu": 1,
+                "gpu_model": "nvidia-tesla-k80",
+                "is_preemptible": False,
+                "is_preemptible_node_required": False,
+            },
+            {
+                "name": "tpu-micro",
+                "cpu": 0.3,
+                "memory_mb": 300,
+                "tpu": {"type": "v2-8", "software_version": "1.14"},
+                "is_preemptible": False,
+                "is_preemptible_node_required": False,
+            },
+        ]
+        return web.Response(status=HTTPNoContent.status_code)
+
+    app = web.Application()
+    app.router.add_put(
+        "/api/v1/clusters/{cluster_name}/orchestrator/resource_presets",
+        update_cluster_resource_presets,
+    )
+
+    srv = await aiohttp_server(app)
+
+    async with make_client(srv.make_url("/api/v1")) as client:
+        await client._admin.update_cluster_resource_presets(
+            "my_cluster",
+            {
+                "cpu-micro": Preset(cpu=0.1, memory_mb=100),
+                "cpu-micro-p": Preset(
+                    cpu=0.1,
+                    memory_mb=100,
+                    is_preemptible=True,
+                    is_preemptible_node_required=True,
+                ),
+                "gpu-micro": Preset(
+                    cpu=0.2, memory_mb=200, gpu=1, gpu_model="nvidia-tesla-k80"
+                ),
+                "tpu-micro": Preset(
+                    cpu=0.3, memory_mb=300, tpu_type="v2-8", tpu_software_version="1.14"
+                ),
+            },
+        )