From 89c59cff2657fe72b97af36f2368791a0e9dfd70 Mon Sep 17 00:00:00 2001 From: Sofia Calgaro Date: Thu, 27 Apr 2023 21:09:26 +0200 Subject: [PATCH 1/2] added std for 'per channel' style --- notebook/L200-plotting-widgets.ipynb | 69 ++++++++++++---- src/legend_data_monitor/plot_styles.py | 27 ++++++- src/legend_data_monitor/plotting.py | 78 +++++++++---------- .../settings/par-settings.json | 10 +-- src/legend_data_monitor/subsystem.py | 29 +++---- 5 files changed, 138 insertions(+), 75 deletions(-) diff --git a/notebook/L200-plotting-widgets.ipynb b/notebook/L200-plotting-widgets.ipynb index 73b0c24..6113f49 100644 --- a/notebook/L200-plotting-widgets.ipynb +++ b/notebook/L200-plotting-widgets.ipynb @@ -32,9 +32,9 @@ "outputs": [], "source": [ "# ------------------------------------------------------------------------------------------ which data do you want to read? CHANGE ME!\n", - "run = \"r002\" # r000, r001, ...\n", + "run = \"r005\" # r000, r001, ...\n", "subsystem = \"geds\" # KEEP 'geds' for the moment\n", - "folder = \"prod-ref\" # you can change me\n", + "folder = \"prod-ref-temp\" # you can change me\n", "period = \"p03\"\n", "version = \"\" # leave an empty string if you're looking at p03 data\n", "\n", @@ -51,7 +51,7 @@ "import ipywidgets as widgets\n", "from IPython.display import display\n", "from matplotlib import pyplot as plt\n", - "from legend_data_monitor import plot_styles, plotting\n", + "from legend_data_monitor import plot_styles, plotting, utils\n", "\n", "%matplotlib widget\n", "\n", @@ -122,8 +122,8 @@ "\n", "# ------------------------------------------------------------------------------------------ get params (based on event type)\n", "evt_type = evt_type_widget.value\n", - "params = list(shelf[\"monitoring\"][evt_type].keys())\n", - "param_widget.options = params\n", + "#params = list(shelf[\"monitoring\"][evt_type].keys())\n", + "param_widget.options = [\"cuspEmax\"]\n", "\n", "print(\"\\033[91mIf you change me, then RUN AGAIN the next cell!!!\\033[0m\")" ] @@ -154,6 +154,19 @@ "print(f\"...data have beeng loaded!\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "b836b69d-b7f5-4131-b6d5-26b637aed57b", + "metadata": {}, + "outputs": [], + "source": [ + "# ------------------------------------------------------------------------------------------ remove problematic dets in cal data\n", + "#df_param = df_param.set_index(\"name\")\n", + "#df_param = df_param.drop(['V01406A', 'V01415A', 'V01387A', 'P00665C', 'P00748B', 'P00748A', 'B00089D'])\n", + "#df_param = df_param.reset_index()" + ] + }, { "cell_type": "markdown", "id": "f1c10c0f-9bed-400f-8174-c6d7e185648b", @@ -190,16 +203,22 @@ "cell_type": "code", "execution_count": null, "id": "2122008e-2a6c-49b6-8a81-d351c1bfd57e", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ "# set plotting options\n", "plot_info[\"plot_style\"] = plot_styles_widget.value\n", + "plot_info[\"plot_structure\"] = plot_structures_widget.value\n", "plot_info[\"resampled\"] = resampled_widget.value\n", "plot_info[\"title\"] = \"\" # for plotting purposes\n", "plot_info[\"subsystem\"] = \"\" # for plotting purposes\n", + "plot_info[\"std\"] = False\n", + "\n", + "df_to_plot = df_param\n", + "\n", + "# turn on the std when plotting individual channels together\n", + "if plot_info[\"plot_structure\"] == \"per channel\":\n", + " plot_info[\"std\"] = True\n", "\n", "if data_format_widget.value == \"absolute values\":\n", " plot_info[\"parameter\"] = (\n", @@ -207,8 +226,9 @@ " if \"_var\" in plot_info[\"parameter\"]\n", " else plot_info[\"parameter\"]\n", " )\n", + " plot_info[\"limits\"] = utils.PLOT_INFO[plot_info[\"parameter\"]][\"limits\"][subsystem][\"absolute\"]\n", " plot_info[\"unit_label\"] = plot_info[\"unit\"]\n", - " if plot_info[\"parameter\"] not in df_param:\n", + " if plot_info[\"parameter\"] not in df_to_plot:\n", " print(\"There is no\", plot_info[\"parameter\"])\n", " sys.exit(\"Stopping notebook.\")\n", "if data_format_widget.value == \"% values\":\n", @@ -217,8 +237,9 @@ " if \"_var\" in plot_info[\"parameter\"]\n", " else plot_info[\"parameter\"] + \"_var\"\n", " )\n", + " plot_info[\"limits\"] = utils.PLOT_INFO[plot_info[\"parameter\"].split(\"_var\")[0]][\"limits\"][subsystem][\"variation\"]\n", " plot_info[\"unit_label\"] = \"%\"\n", - " if plot_info[\"parameter\"] not in df_param:\n", + " if plot_info[\"parameter\"] not in df_to_plot:\n", " print(\"There is no\", plot_info[\"parameter\"])\n", " sys.exit(\"Stopping notebook.\")\n", "\n", @@ -227,20 +248,20 @@ " for string in [1, 2, 3, 4, 5, 7, 8, 9, 10, 11]:\n", " if plot_structures_widget.value == \"per channel\":\n", " plotting.plot_per_ch(\n", - " df_param[df_param[\"location\"] == string], plot_info, \"\"\n", + " df_to_plot[df_to_plot[\"location\"] == string], plot_info, \"\"\n", " ) # plot one canvas per channel\n", " elif plot_structures_widget.value == \"per string\":\n", " plotting.plot_per_string(\n", - " df_param[df_param[\"location\"] == string], plot_info, \"\"\n", + " df_to_plot[df_to_plot[\"location\"] == string], plot_info, \"\"\n", " ) # plot one canvas per string\n", "else: # let's get one string in output\n", " if plot_structures_widget.value == \"per channel\":\n", " plotting.plot_per_ch(\n", - " df_param[df_param[\"location\"] == strings_widget.value], plot_info, \"\"\n", + " df_to_plot[df_to_plot[\"location\"] == strings_widget.value], plot_info, \"\"\n", " ) # plot one canvas per channel\n", " elif plot_structures_widget.value == \"per string\":\n", " plotting.plot_per_string(\n", - " df_param[df_param[\"location\"] == strings_widget.value], plot_info, \"\"\n", + " df_to_plot[df_to_plot[\"location\"] == strings_widget.value], plot_info, \"\"\n", " ) # plot one canvas per string" ] }, @@ -279,7 +300,25 @@ ] } ], - "metadata": {}, + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + } + }, "nbformat": 4, "nbformat_minor": 5 } diff --git a/src/legend_data_monitor/plot_styles.py b/src/legend_data_monitor/plot_styles.py index 04b4d74..01a1adf 100644 --- a/src/legend_data_monitor/plot_styles.py +++ b/src/legend_data_monitor/plot_styles.py @@ -9,7 +9,7 @@ from matplotlib.axes import Axes from matplotlib.dates import DateFormatter, date2num, num2date from matplotlib.figure import Figure -from pandas import DataFrame, Timedelta +from pandas import DataFrame, Timedelta, concat from . import utils @@ -39,6 +39,7 @@ def plot_vs_time( data_channel[plot_info["parameter"]], zorder=0, color=all_col, + linewidth=1, ) # ------------------------------------------------------------------------- @@ -48,6 +49,7 @@ def plot_vs_time( if plot_info["resampled"] != "no": # unless event rate - already resampled and counted in some time window if not plot_info["parameter"] == "event_rate": + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 1 - resampling # resample in given time window, as start pick the first timestamp in table resampled = ( data_channel.set_index("datetime") @@ -67,10 +69,26 @@ def plot_vs_time( resampled[plot_info["parameter"]], color=res_col, zorder=1, - marker="o", + #marker="o", linestyle="-", ) + # evaluation of std bands, if enabled + if plot_info["std"] is True: + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 2 - std evaluation + std_data = ( + data_channel.set_index("datetime") + .resample(plot_info["time_window"], origin="start") + .std(numeric_only=True) + ) + std_data = std_data.reset_index() + + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 3 - appending std to the resampled dataframe + std_data = std_data.rename(columns={plot_info["parameter"] : "std"}) + new_dataframe = concat([resampled, std_data[['std']]], ignore_index=False, axis=1) + + ax.fill_between(resampled["datetime"].dt.to_pydatetime(), resampled[plot_info["parameter"]] - new_dataframe['std'], resampled[plot_info["parameter"]] + new_dataframe['std'], alpha=0.25, color=res_col) + # ------------------------------------------------------------------------- # beautification # ------------------------------------------------------------------------- @@ -222,6 +240,11 @@ def plot_scatter( ) fig.supylabel(y_label) + # plot the position of the two K lines + if plot_info["parameter"] == "K_events": + ax.axhline(y=1460.822, color="gray", linestyle="--") + ax.axhline(y=1524.6, color="gray", linestyle="--") + # saving x,y data into output files ch_dict = { "values": {"all": data_channel[plot_info["parameter"]], "resampled": []}, diff --git a/src/legend_data_monitor/plotting.py b/src/legend_data_monitor/plotting.py index 47c9a57..4dc5349 100644 --- a/src/legend_data_monitor/plotting.py +++ b/src/legend_data_monitor/plotting.py @@ -126,6 +126,9 @@ def make_subsystem_plots( plot_settings["resampled"] if "resampled" in plot_settings else "" ) + # information for shifting the channels or not (not needed only for the 'per channel' structure option) when plotting the std + plot_info["std"] = True if plot_settings["plot_structure"] == "per channel" else False + if plot_settings["plot_style"] == "vs time": if plot_info["resampled"] == "": plot_info["resampled"] = "also" @@ -312,9 +315,8 @@ def plot_per_ch(data_analysis: DataFrame, plot_info: dict, pdf: PdfPages): # remove automatic y label since there will be a shared one axes[ax_idx].set_ylabel("") - # plot line at 0% for variation - if plot_info["unit_label"] == "%": - axes[ax_idx].axhline(y=0, color="gray", linestyle="--") + # plot limits + plot_limits(axes[ax_idx], plot_info["limits"]) ax_idx += 1 @@ -327,10 +329,7 @@ def plot_per_ch(data_analysis: DataFrame, plot_info: dict, pdf: PdfPages): axes[0].set_title(f"{plot_info['locname']} {location}") fig.suptitle(f"{plot_info['subsystem']} - {plot_info['title']}", y=y_title) - if pdf: - plt.savefig(pdf, format="pdf", bbox_inches="tight") - # figures are retained until explicitly closed; close to not consume too much memory - plt.close() + save_pdf(plt, pdf) return fig @@ -403,28 +402,20 @@ def plot_per_cc4(data_analysis: DataFrame, plot_info: dict, pdf: PdfPages): axes[ax_idx].set_ylabel("") axes[ax_idx].legend(labels=labels, loc="center left", bbox_to_anchor=(1, 0.5)) - # plot the position of the two K lines - if plot_info["parameter"] == "K_events": - axes[ax_idx].axhline(y=1460.822, color="gray", linestyle="--") - axes[ax_idx].axhline(y=1524.6, color="gray", linestyle="--") + # plot limits + plot_limits(axes[ax_idx], plot_info["limits"]) - # plot line at 0% for variation - if plot_info["unit_label"] == "%": - axes[ax_idx].axhline(y=0, color="gray", linestyle="--") ax_idx += 1 # ------------------------------------------------------------------------------- y_title = 1.05 if plot_info["subsystem"] == "pulser" else 1.01 fig.suptitle(f"{plot_info['subsystem']} - {plot_info['title']}", y=y_title) - # if no pdf is specified, then the function is not being called by make_subsystem_plots() - if pdf: - plt.savefig(pdf, format="pdf", bbox_inches="tight") - # figures are retained until explicitly closed; close to not consume too much memory - plt.close() + save_pdf(plt, pdf) return fig +import numpy as np def plot_per_string(data_analysis: DataFrame, plot_info: dict, pdf: PdfPages): # --- choose plot function based on user requested style e.g. vs time or histogram plot_style = plot_styles.PLOT_STYLE[plot_info["plot_style"]] @@ -481,8 +472,13 @@ def plot_per_string(data_analysis: DataFrame, plot_info: dict, pdf: PdfPages): col_idx = 0 labels = [] for label, data_channel in data_location.groupby("label"): + entries = data_channel[plot_info["parameter"]] + entries_avg = np.mean(entries) + rms_ch = np.sqrt(np.mean(np.square(entries - entries_avg))) + FWHM_ch = 2.355*rms_ch + _ = plot_style(data_channel, fig, axes[ax_idx], plot_info, COLORS[col_idx]) - labels.append(label) + labels.append(label+f" - FWHM: {round(FWHM_ch, 2)}") col_idx += 1 # add grid @@ -493,25 +489,16 @@ def plot_per_string(data_analysis: DataFrame, plot_info: dict, pdf: PdfPages): axes[ax_idx].set_ylabel("") axes[ax_idx].legend(labels=labels, loc="center left", bbox_to_anchor=(1, 0.5)) - # plot the position of the two K lines - if plot_info["parameter"] == "K_events": - axes[ax_idx].axhline(y=1460.822, color="gray", linestyle="--") - axes[ax_idx].axhline(y=1524.6, color="gray", linestyle="--") + # plot limits + plot_limits(axes[ax_idx], plot_info["limits"]) - # plot line at 0% for variation - if plot_info["unit_label"] == "%": - axes[ax_idx].axhline(y=0, color="gray", linestyle="--") ax_idx += 1 # ------------------------------------------------------------------------------- y_title = 1.05 if plot_info["subsystem"] == "pulser" else 1.01 fig.suptitle(f"{plot_info['subsystem']} - {plot_info['title']}", y=y_title) - # if no pdf is specified, then the function is not being called by make_subsystem_plots() - if pdf: - plt.savefig(pdf, format="pdf", bbox_inches="tight") - # figures are retained until explicitly closed; close to not consume too much memory - plt.close() + save_pdf(plt, pdf) return fig @@ -637,14 +624,9 @@ def plot_array(data_analysis: DataFrame, plot_info: dict, pdf: PdfPages): fig.supxlabel("") fig.suptitle(f"{plot_info['subsystem']} - {plot_info['title']}", y=1.05) - # ------------------------------------------------------------------------------- - # if no pdf is specified, then the function is not being called by make_subsystem_plots() - if pdf: - plt.savefig(pdf, format="pdf", bbox_inches="tight") - # figures are retained until explicitly closed; close to not consume too much memory - plt.close() + save_pdf(plt, pdf) - # return fig + return fig # ------------------------------------------------------------------------------- @@ -803,6 +785,24 @@ def plot_per_barrel_and_position( return par_dict +# ------------------------------------------------------------------------------- +# plotting functions +# ------------------------------------------------------------------------------- + +def plot_limits(ax: plt.Axes, limits: dict): + """Plot limits (if present) on the plot.""" + if not all([x is None for x in limits]): + if limits[0] is not None: + ax.axhline(y=limits[0], color="red", linestyle="--") + if limits[1] is not None: + ax.axhline(y=limits[1], color="red", linestyle="--") + +def save_pdf(plt, pdf: PdfPages): + """Save the plot to a PDF file. The plot is closed after saving.""" + if pdf: + plt.savefig(pdf, format="pdf", bbox_inches="tight") + plt.close() + # ------------------------------------------------------------------------------- # mapping user keywords to plot style functions diff --git a/src/legend_data_monitor/settings/par-settings.json b/src/legend_data_monitor/settings/par-settings.json index e753e32..2ade9d8 100644 --- a/src/legend_data_monitor/settings/par-settings.json +++ b/src/legend_data_monitor/settings/par-settings.json @@ -129,7 +129,7 @@ "absolute": [null, null] }, "geds": { - "variation": [-0.25, 0.25], + "variation": [-0.025, 0.025], "absolute": [null, null] } } @@ -144,7 +144,7 @@ "absolute": [null, null] }, "geds": { - "variation": [-0.25, 0.25], + "variation": [-0.025, 0.025], "absolute": [null, null] } } @@ -459,7 +459,7 @@ "absolute": [null, null] }, "geds": { - "variation": [-0.25, 0.25], + "variation": [-0.025, 0.025], "absolute": [null, null] } } @@ -474,7 +474,7 @@ "absolute": [null, null] }, "geds": { - "variation": [-0.25, 0.25], + "variation": [-0.025, 0.025], "absolute": [null, null] } } @@ -489,7 +489,7 @@ "absolute": [null, null] }, "geds": { - "variation": [-0.25, 0.25], + "variation": [-0.025, 0.025], "absolute": [null, null] } } diff --git a/src/legend_data_monitor/subsystem.py b/src/legend_data_monitor/subsystem.py index ab92de6..4f4fc00 100644 --- a/src/legend_data_monitor/subsystem.py +++ b/src/legend_data_monitor/subsystem.py @@ -559,35 +559,36 @@ def construct_dataloader_configs(self, params: list_of_str): # set up tiers depending on what parameters we need # ------------------------------------------------------------------------- - # ronly load channels that are on (off channels will crash DataLoader) + # only load channels that are on (off channels will crash DataLoader) chlist = list(self.channel_map[self.channel_map["status"] == "on"]["channel"]) removed_chs = list( - self.channel_map[self.channel_map["status"] == "off"]["channel"] + self.channel_map[self.channel_map["status"] == "off"]["name"] ) - utils.logger.info(f"...... not loading channels with status off: {removed_chs}") - # for L60-p01 and L200-p02, keep using 3 digits - if int(self.period[-1]) < 3: - ch_format = "ch:03d" - # from L200-p03 included, uses 7 digits + # remove p03 channels who are not properly behaving in calib data (from George's analysis) if int(self.period[-1]) >= 3: - ch_format = "ch:07d" + names = ["V01406A", "V01415A", "V01387A", "P00665C", "P00748B", "P00748A"]#, "B00089D"] + probl_dets = [] + for name in names: + probl_det = list(self.channel_map[self.channel_map["name"] == name]["channel"]) + # the following 'if' is needed to avoid errors when setting up 'pulser' + if probl_det != []: + probl_dets.append(probl_det[0]) + if probl_dets != []: + utils.logger.info(f"...... not loading problematic detectors for {self.period}: {names}") + chlist = [ch for ch in chlist if ch not in probl_dets] # --- settings for each tier for tier, tier_params in param_tiers.groupby("tier"): dict_dbconfig["tier_dirs"][tier] = f"/{tier}" # type not fixed and instead specified in the query dict_dbconfig["file_format"][tier] = ( - "/{type}/" - + self.period # {period} - + "/{run}/{exp}-" - + self.period # {period} - + "-{run}-{type}-{timestamp}-tier_" + "/{type}/{period}/{run}/{exp}-{period}-{run}-{type}-{timestamp}-tier_" + tier + ".lh5" ) - dict_dbconfig["table_format"][tier] = "ch{" + ch_format + "}/" + tier + dict_dbconfig["table_format"][tier] = "ch{ch:03d}/" + tier dict_dbconfig["tables"][tier] = chlist From 6c9dd0868da31a80949a870c71304e0a33d6fc33 Mon Sep 17 00:00:00 2001 From: Sofia Calgaro <77326044+sofia-calgaro@users.noreply.github.com> Date: Thu, 27 Apr 2023 21:12:23 +0200 Subject: [PATCH 2/2] fixing channels --- src/legend_data_monitor/subsystem.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/legend_data_monitor/subsystem.py b/src/legend_data_monitor/subsystem.py index 4f4fc00..29c3e45 100644 --- a/src/legend_data_monitor/subsystem.py +++ b/src/legend_data_monitor/subsystem.py @@ -579,16 +579,27 @@ def construct_dataloader_configs(self, params: list_of_str): utils.logger.info(f"...... not loading problematic detectors for {self.period}: {names}") chlist = [ch for ch in chlist if ch not in probl_dets] + # for L60-p01 and L200-p02, keep using 3 digits + if int(self.period[-1]) < 3: + ch_format = "ch:03d" + # from L200-p03 included, uses 7 digits + if int(self.period[-1]) >= 3: + ch_format = "ch:07d" + # --- settings for each tier for tier, tier_params in param_tiers.groupby("tier"): dict_dbconfig["tier_dirs"][tier] = f"/{tier}" # type not fixed and instead specified in the query dict_dbconfig["file_format"][tier] = ( - "/{type}/{period}/{run}/{exp}-{period}-{run}-{type}-{timestamp}-tier_" + "/{type}/" + + self.period # {period} + + "/{run}/{exp}-" + + self.period # {period} + + "-{run}-{type}-{timestamp}-tier_" + tier + ".lh5" ) - dict_dbconfig["table_format"][tier] = "ch{ch:03d}/" + tier + dict_dbconfig["table_format"][tier] = "ch{" + ch_format + "}/" + tier dict_dbconfig["tables"][tier] = chlist