feat: add cluster-wide message (#9261)

Co-authored-by: eecsliu <[email protected]>
Co-authored-by: Tara Charter <[email protected]>
Co-authored-by: Keita Fish <[email protected]>

docs/release-notes/cluster-message.rst

@@ -0,0 +1,9 @@
+:orphan:
+
+**New Features**
+
+-  WebUI: Add the ability for administrators to use the CLI to set a message to be displayed on all
+   pages of the WebUI (for example, ``det master cluster-message set -m "Your message"``). Optional
+   flags are available for scheduling the message with a start time and an end time. Administrators
+   can clear the message anytime using ``det master cluster-message clear``. Only one message can be
+   active at a time, so setting a new message will replace the previous one.

docs/tools/webui-if.rst

@@ -168,3 +168,49 @@ You can start :ref:`notebooks` from the WebUI.
 ***********************
 
 You can launch TensorBoard from the WebUI. To learn how, visit :ref:`tensorboards`.
+
+*****************************
+ Displaying a Banner Message
+*****************************
+
+Administrators can create a banner message to alert users about important information, such as
+maintenance, setting a password, or other announcements. This message will be displayed on the
+header of every page in the WebUI for the configured duration. Commands include ``help``, ``clear``,
+``get``, and ``set``.
+
+**Prerequisites**
+
+-  Install the :ref:`CLI <cli-ug>`.
+
+**Prepare the Message**
+
+Prepare the maintenance message using the CLI command, ``det master cluster-message set``.
+
+-  For example, the following command creates a maintenance message with a start and end date (which
+   must be expressed in UTC):
+
+      .. code:: bash
+
+         det master cluster-message set --message "Scheduled maintenance on Dec 1st from 10pm CST to 11pm CST." --start "2024-12-02-04:00:00Z" --end "2024-12-02-05:00:00Z"
+
+-  You can also express the end date as a duration:
+
+      .. code:: bash
+
+         det master cluster-message set --message "Please change your password by Jan 1, 2025" --duration 14d
+
+**Verify the Message**
+
+Verify the message with the following command:
+
+   .. code:: bash
+
+      det master cluster-message get
+
+**Clear the Message**
+
+Clear the message with the following command:
+
+   .. code:: bash
+
+      det master cluster-message clear

e2e_tests/tests/cluster/test_cluster_message.py

@@ -0,0 +1,134 @@
+import datetime
+
+import pytest
+
+from tests import api_utils, detproc
+
+
+@pytest.mark.e2e_cpu
+def test_cluster_message_when_admin() -> None:
+    admin = api_utils.admin_session()
+
+    # Clear and make sure get throws no errors and returns no cluster message
+    detproc.check_call(
+        admin,
+        ["det", "master", "cluster-message", "clear"],
+    )
+    res = detproc.check_json(admin, ["det", "master", "cluster-message", "get", "--json"])
+    assert res["clusterMessage"] is None, "there should not be a cluster message set after clearing"
+
+    # Python's ISO format isn't RFC 3339, so it doesn't include the Z; add it
+    end_time = (datetime.datetime.utcnow() + datetime.timedelta(days=500)).isoformat() + "Z"
+    test_message = "test message 1"
+
+    # Set the message
+    detproc.check_call(
+        admin,
+        ["det", "master", "cluster-message", "set", "-m", test_message, "--end", end_time],
+    )
+
+    # Make sure master info has an active cluster message
+    master_info = detproc.check_json(
+        admin,
+        ["det", "master", "info", "--json"],
+    )
+    assert (
+        master_info["clusterMessage"] is not None
+    ), "active cluster message should be visible in master info"
+
+    # Check the result
+    actual_message = detproc.check_json(
+        admin,
+        ["det", "master", "cluster-message", "get", "--json"],
+    )["clusterMessage"]
+    assert (
+        actual_message["message"] == test_message
+    ), "the message returned by the API was not the one expected"
+    assert (
+        actual_message["endTime"] == end_time
+    ), "the end time returned by the API was not what was expected"
+
+    # Test setting one with --duration
+    test_message = "test message 2"
+    start_time = "2035-01-01T00:00:00Z"
+    duration = "22h"
+    expected_end = "2035-01-01T22:00:00Z"
+    detproc.check_call(
+        admin,
+        [
+            "det",
+            "master",
+            "cluster-message",
+            "set",
+            "-m",
+            test_message,
+            "--start",
+            start_time,
+            "--duration",
+            duration,
+        ],
+    )
+
+    # Make sure master info has no active cluster message since it's scheduled in the future
+    master_info = detproc.check_json(
+        admin,
+        ["det", "master", "info", "--json"],
+    )
+    assert (
+        master_info["clusterMessage"] is None
+    ), "cluster message scheduled in the future should not be visible in master info"
+
+    # Make sure cluster message *is* present in result of cluster-message get
+    msg = detproc.check_json(admin, ["det", "master", "cluster-message", "get", "--json"])[
+        "clusterMessage"
+    ]
+    assert (
+        msg["message"] == test_message
+    ), "cluster message returned by the API was not the one expected"
+    assert (
+        msg["endTime"] == expected_end
+    ), "cluster message end time returned by the API was not the one expected"
+
+    # Clear and make sure the cluster message is unset
+    detproc.check_call(
+        admin,
+        ["det", "master", "cluster-message", "clear"],
+    )
+    resp = detproc.check_json(admin, ["det", "master", "cluster-message", "get", "--json"])
+    assert (
+        resp["clusterMessage"] is None
+    ), "there should not be a cluster message set after clearing"
+    master_info = detproc.check_json(
+        admin,
+        ["det", "master", "info", "--json"],
+    )
+    assert (
+        master_info["clusterMessage"] is None
+    ), "cluster message should not be visible in master info after clearing"
+
+
+@pytest.mark.e2e_cpu
+def test_cluster_message_requires_admin() -> None:
+    user = api_utils.user_session()
+    admin = api_utils.admin_session()
+
+    # Stuff that should fail when not admin
+    proc = detproc.run(user, ["det", "master", "cluster-message", "get"])
+    assert proc.returncode != 0, "cluster message get should have failed when not admin"
+    proc = detproc.run(user, ["det", "master", "cluster-message", "set", "-m", "foobarbaz"])
+    assert proc.returncode != 0, "cluster message set should have failed when not admin"
+    proc = detproc.run(user, ["det", "master", "cluster-message", "clear"])
+    assert proc.returncode != 0, "cluster message clear should have failed when not admin"
+
+    # Actually set a cluster message as admin
+    expected_message = "foobarbaz"
+    detproc.check_call(admin, ["det", "master", "cluster-message", "set", "-m", expected_message])
+
+    # Verify we see the correct message as a normal user
+    master_info = detproc.check_json(user, ["det", "master", "info", "--json"])
+    assert (
+        master_info["clusterMessage"] is not None
+    ), "active cluster messages should be visible to non-admins"
+
+    # Clean up after ourselves
+    detproc.run(admin, ["det", "master", "cluster-message", "clear"])

harness/determined/cli/master.py

@@ -1,4 +1,5 @@
 import argparse
+import datetime
 from typing import Any, List, Optional
 
 from determined import cli
@@ -70,6 +71,39 @@ def logs(args: argparse.Namespace) -> None:
         print(format_log_entry(response.logEntry))
 
 
+def set_message(args: argparse.Namespace) -> None:
+    sess = cli.setup_session(args)
+
+    if args.message is None:
+        raise ValueError("Provide a message using the -m flag.")
+    if args.start is not None and not util.is_protobuf_timestamp(args.start):
+        raise ValueError("Start time must be RFC-3339, i.e. of the form YYYY-MM-DDThh:mm:ssZ")
+    if args.end is not None and not util.is_protobuf_timestamp(args.end):
+        raise ValueError("End time must be RFC-3339, i.e. of the form YYYY-MM-DDThh:mm:ssZ")
+
+    body = bindings.v1SetClusterMessageRequest(
+        startTime=args.start, endTime=args.end, message=args.message, duration=args.duration
+    )
+    bindings.put_SetClusterMessage(sess, body=body)
+
+
+def clear_cluster_message(args: argparse.Namespace) -> None:
+    sess = cli.setup_session(args)
+    bindings.delete_DeleteClusterMessage(sess)
+
+
+def get_cluster_message(args: argparse.Namespace) -> None:
+    sess = cli.setup_session(args)
+
+    resp = bindings.get_GetClusterMessage(sess)
+    message = resp.to_json()
+
+    if args.json:
+        render.print_json(message)
+    else:
+        print(util.yaml_safe_dump(message, default_flow_style=False))
+
+
 # fmt: off
 
 args_description = [
@@ -125,6 +159,31 @@ def logs(args: argparse.Namespace) -> None:
                 help="number of lines to show, counting from the end "
                 "of the log (default is all)")
         ]),
+        cli.Cmd("cluster-message", None, "set or clear cluster-wide message", [
+            cli.Cmd("set", set_message, "create or edit the displayed cluster-wide message", [
+                cli.Arg("-s", "--start", default=datetime.datetime.utcnow().isoformat("T") + "Z",
+                    help="Timestamp to start displaying message (RFC 3339 format), "
+                    + "e.g. '2021-10-26T23:17:12Z'; default is now."),
+                cli.Group(
+                    cli.Arg("-e", "--end", default=None,
+                            help="Timestamp to end displaying message (RFC 3339 format), "
+                            + "e.g. '2021-10-26T23:17:12Z'; default is indefinite."),
+                    cli.Arg("-d", "--duration", default=None,
+                            help="How long the message should last; mutually exclusive with "
+                            + "--end and should be formatted as a Go duration string "
+                            + "e.g. 24h, 2w, 5d"),
+                ),
+                cli.Arg("-m", "--message", default=None,
+                        help="Text of the message to display to users"),
+            ]),
+            cli.Cmd("clear", clear_cluster_message, "clear cluster-wide message", [
+                cli.Arg("-c", "--clear", action="store_true", default=False,
+                    help="Clear all cluster-wide message"),
+            ]),
+            cli.Cmd("get", get_cluster_message, "get cluster-wide message", [
+                cli.Group(cli.output_format_args["json"], cli.output_format_args["yaml"])
+            ]),
+        ])
     ])
 ]  # type: List[Any]