Merge pull request #50 from lsst/tickets/PREOPS-3846

Tickets/preops 3846

schedview/app/prenight/prenight.py

@@ -918,10 +918,10 @@ def __init__(self, data_dir=None, **kwargs):
 
         # make a dictionary of refereneces to the param paths, so that
         # they can be updated by key.
-        path_for_kwargs = {
-            "opsim_db": self.param["opsim_output_fname"].path,
-            "scheduler": self.param["scheduler_fname"].path,
-            "reward": self.param["rewards_fname"].path,
+        fname_params = {
+            "opsim_db": self.param["opsim_output_fname"],
+            "scheduler": self.param["scheduler_fname"],
+            "reward": self.param["rewards_fname"],
         }
 
         # In cases where the caller has not specified a value, set
@@ -935,11 +935,11 @@ def __init__(self, data_dir=None, **kwargs):
             }
 
         # Actually assign the names or globs to the path references.
-        for arg_name in path_for_kwargs:
+        for arg_name in fname_params:
             if arg_name in kwargs:
-                path_for_kwargs[arg_name] = kwargs[arg_name]
+                fname_params[arg_name].update(path=kwargs[arg_name])
             elif data_dir is not None:
-                path_for_kwargs[arg_name] = fname_glob[arg_name]
+                fname_params[arg_name].update(path=fname_glob[arg_name])
 
 
 def prenight_app(*args, **kwargs):
@@ -1125,7 +1125,7 @@ def prenight_app_with_params():
         return prenight_app(**prenight_app_parameters)
 
     pn.serve(
-        prenight_app_with_params,
+        {"schedview-prenight": prenight_app_with_params},
         port=prenight_port,
         title="Prenight Dashboard",
         show=show,

schedview/app/scheduler_dashboard/scheduler_dashboard.py

@@ -23,11 +23,16 @@
 
 """schedview docstring"""
 
+import argparse
 import importlib.resources
 import logging
 import os
 import traceback
+
+# Filter the astropy warnings swamping the terminal
+import warnings
 from datetime import datetime
+from glob import glob
 from zoneinfo import ZoneInfo
 
 import bokeh
@@ -36,6 +41,7 @@
 import param
 import rubin_sim.site_models
 from astropy.time import Time
+from astropy.utils.exceptions import AstropyWarning
 from bokeh.models import ColorBar, LinearColorMapper
 from bokeh.models.widgets.tables import BooleanFormatter, HTMLTemplateFormatter
 from pandas import Timestamp
@@ -49,12 +55,18 @@
 import schedview.collect.scheduler_pickle
 import schedview.compute.scheduler
 import schedview.compute.survey
+import schedview.param
 import schedview.plot.survey
 
+# Filter astropy warning that's filling the terminal with every update.
+warnings.filterwarnings("ignore", category=AstropyWarning)
+
 DEFAULT_CURRENT_TIME = Time.now()
 DEFAULT_TIMEZONE = "UTC"  # "America/Santiago"
 LOGO = "/assets/lsst_white_logo.png"
 COLOR_PALETTES = [color for color in bokeh.palettes.__palettes__ if "256" in color]
+PACKAGE_DATA_DIR = importlib.resources.files("schedview.data").as_posix()
+USDF_DATA_DIR = "/sdf/group/rubin/web_data/sim-data/schedview"
 
 
 pn.extension(
@@ -1274,6 +1286,33 @@ def map_title(self):
         return self.map_title_pane
 
 
+class RestrictedFilesScheduler(Scheduler):
+    """A Parametrized container for parameters, data, and panel objects for the
+    scheduler dashboard.
+    """
+
+    # Param parameters that are modifiable by user actions.
+    scheduler_fname_doc = """URL or file name of the scheduler pickle file.
+    Such a pickle file can either be of an instance of a subclass of
+    rubin_sim.scheduler.schedulers.CoreScheduler, or a tuple of the form
+    (scheduler, conditions), where scheduler is an instance of a subclass of
+    rubin_sim.scheduler.schedulers.CoreScheduler, and conditions is an
+    instance of rubin_sim.scheduler.conditions.Conditions.
+    """
+    scheduler_fname = schedview.param.FileSelectorWithEmptyOption(
+        path=f"{PACKAGE_DATA_DIR}/*scheduler*.p*",
+        doc=scheduler_fname_doc,
+        default=None,
+        allow_None=True,
+    )
+
+    def __init__(self, data_dir=None):
+        super().__init__()
+
+        if data_dir is not None:
+            self.param["scheduler_fname"].update(path=f"{data_dir}/*scheduler*.p*")
+
+
 # --------------------------------------------------------------- Key functions
 
 
@@ -1449,7 +1488,7 @@ def generate_key():
 # ------------------------------------------------------------ Create dashboard
 
 
-def scheduler_app(date_time=None, scheduler_pickle=None):
+def scheduler_app(date_time=None, scheduler_pickle=None, **kwargs):
     """Create a dashboard with grids of Param parameters, Tabulator widgets,
     and Bokeh plots.
 
@@ -1471,15 +1510,57 @@ def scheduler_app(date_time=None, scheduler_pickle=None):
         max_height=1000,
     ).servable()
 
-    scheduler = Scheduler()
+    from_urls = False
+    data_dir = None
+
+    if "data_from_urls" in kwargs.keys():
+        from_urls = kwargs["data_from_urls"]
+        del kwargs["data_from_urls"]
+
+    if "data_dir" in kwargs.keys():
+        data_dir = kwargs["data_dir"]
+        del kwargs["data_dir"]
+
+    scheduler = None
+    data_loading_widgets = {}
+    # Accept pickle files from url or any path.
+    if from_urls:
+        scheduler = Scheduler()
+        # read pickle and time if provided to the function in a notebook
+        # it will be overriden if the dashboard runs in an app
+        if date_time is not None:
+            scheduler.widget_datetime = date_time
+
+        if scheduler_pickle is not None:
+            scheduler.scheduler_fname = scheduler_pickle
+
+        # Sync url parameters only if the files aren't restricted.
+        if pn.state.location is not None:
+            pn.state.location.sync(
+                scheduler,
+                {
+                    "scheduler_fname": "scheduler",
+                    "nside": "nside",
+                    "url_mjd": "mjd",
+                },
+            )
 
-    if date_time is not None:
-        scheduler.widget_datetime = date_time
+        data_loading_widgets = {
+            "scheduler_fname": {
+                # "widget_type": pn.widgets.TextInput,
+                "placeholder": "filepath or URL of pickle",
+            },
+            "widget_datetime": pn.widgets.DatetimePicker,
+        }
 
-    if scheduler_pickle is not None:
-        scheduler.scheduler_fname = scheduler_pickle
+    # Restrict files to data_directory.
+    else:
+        scheduler = RestrictedFilesScheduler(data_dir=data_dir)
+        data_loading_widgets = {
+            "widget_datetime": pn.widgets.DatetimePicker,
+        }
 
-    # show dashboard as busy when scheduler.show_loading_spinner is True
+    # Show dashboard as busy when scheduler.show_loading_spinner is True.
     @pn.depends(loading=scheduler.param.show_loading_indicator, watch=True)
     def update_loading(loading):
         start_loading_spinner(sched_app) if loading else stop_loading_spinner(sched_app)
@@ -1510,15 +1591,10 @@ def update_loading(loading):
     sched_app[8:33, 0:21] = pn.Param(
         scheduler,
         parameters=["scheduler_fname", "widget_datetime", "widget_tier"],
-        widgets={
-            "scheduler_fname": {
-                "widget_type": pn.widgets.TextInput,
-                "placeholder": "filepath or URL of pickle",
-            },
-            "widget_datetime": pn.widgets.DatetimePicker,
-        },
+        widgets=data_loading_widgets,
         name="Select pickle file, date and tier.",
     )
+
     # Survey rewards table and header.
     sched_app[8:33, 21:67] = pn.Row(
         pn.Spacer(width=10),
@@ -1579,22 +1655,43 @@ def update_loading(loading):
         collapsed=True,
     )
 
-    # sync URL parameters to scheduler params
-    if pn.state.location is not None:
-        pn.state.location.sync(
-            scheduler,
-            {
-                "scheduler_fname": "scheduler",
-                "nside": "nside",
-                "url_mjd": "mjd",
-            },
-        )
-
     return sched_app
 
 
+def parse_arguments():
+    """
+    Parse commandline arguments to read data directory if provided
+    """
+    parser = argparse.ArgumentParser(description="On-the-fly Rubin Scheduler dashboard")
+    default_data_dir = f"{USDF_DATA_DIR}/*" if os.path.exists(USDF_DATA_DIR) else PACKAGE_DATA_DIR
+
+    parser.add_argument(
+        "--data_dir",
+        "-d",
+        type=str,
+        default=default_data_dir,
+        help="The base directory for data files.",
+    )
+
+    parser.add_argument(
+        "--data_from_urls",
+        action="store_true",
+        help="Let the user specify URLs from which to load data. THIS IS NOT SECURE.",
+    )
+
+    args = parser.parse_args()
+
+    if len(glob(args.data_dir)) == 0 and not args.data_from_urls:
+        args.data_dir = PACKAGE_DATA_DIR
+
+    scheduler_app_params = args.__dict__
+
+    return scheduler_app_params
+
+
 def main():
     print("Starting scheduler dashboard.")
+    commandline_args = parse_arguments()
 
     if "SCHEDULER_PORT" in os.environ:
         scheduler_port = int(os.environ["SCHEDULER_PORT"])
@@ -1603,8 +1700,11 @@ def main():
 
     assets_dir = os.path.join(importlib.resources.files("schedview"), "app", "scheduler_dashboard", "assets")
 
+    def scheduler_app_with_params():
+        return scheduler_app(**commandline_args)
+
     pn.serve(
-        scheduler_app,
+        scheduler_app_with_params,
         port=scheduler_port,
         title="Scheduler Dashboard",
         show=True,

schedview/param.py

@@ -1,9 +1,12 @@
 # Subclasses of param.Parameter for use in schedview.
 
 import glob
+import os
+import pathlib
 
 import pandas as pd
 import param
+from param import Undefined
 
 
 class Series(param.Parameter):
@@ -88,11 +91,16 @@ class FileSelectorWithEmptyOption(param.FileSelector):
     Like param.FileSelector, but allows None to be deliberately selected.
     """
 
-    def update(self, path=None):
-        if path is not None and path != self.path:
-            self.path = path
-
-        self.objects = [""] + sorted(glob.glob(self.path))
+    def update(self, path=Undefined):
+        if path is Undefined:
+            path = self.path
+        if path == "":
+            self.objects = []
+        else:
+            # Convert using os.fspath and pathlib.Path to handle ensure
+            # the path separators are consistent (on Windows in particular)
+            pathpattern = os.fspath(pathlib.Path(path))
+            self.objects = [""] + sorted(glob.glob(pathpattern))
         if self.default in self.objects:
             return
         self.default = self.objects[0] if self.objects else None

tests/test_scheduler_dashboard.py

@@ -70,7 +70,7 @@ def setUp(self) -> None:
         bokeh.io.reset_output()
 
     def test_scheduler_app(self):
-        sched_app = scheduler_app(date_time=TEST_DATE, scheduler_pickle=TEST_PICKLE)
+        sched_app = scheduler_app(date_time=TEST_DATE, scheduler_pickle=TEST_PICKLE, data_from_urls=True)
         sched_app_bokeh_model = sched_app.get_root()
         with TemporaryDirectory() as scheduler_dir:
             sched_out_path = Path(scheduler_dir)