Picasso server REV

.github/workflows/main.yml

@@ -16,10 +16,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python 3.7
+    - name: Set up Python 3.8
       uses: actions/setup-python@v1
       with:
-        python-version: 3.7
+        python-version: 3.8
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/pypi_test.yml

@@ -0,0 +1,36 @@
+on:
+  push:
+    branches: [ main ]
+  workflow_dispatch:
+
+name: Default installation and tests
+
+jobs:
+  install:
+    name: Test on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macOS-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          python-version: ${{ matrix.python-version }}
+      - name: Conda info
+        shell: bash -l {0}
+        run: conda info
+      - name: Test pip installation 
+        shell: bash -l {0}
+        run: |
+          conda create -n picasso python=3.8 -y
+          conda activate picasso
+          pip install picassosr
+          conda deactivate
+      - name: Testing modules
+        shell: bash -l {0}
+        run: |
+          conda activate picasso
+          picasso
+          conda deactivate

docs/server.rst

@@ -77,7 +77,7 @@ To compare experiments, select one or multiple experiments from the dropdown men
 If multiple hdf files are present, you can select hdf files that belong to the same file family.
 
 **Comparing files will load the entire hdf file and could mean that one is comparing millions of localizations.
-Creating the plots might, therefore, not be instantaneous. **
+Creating the plots might, therefore, not be instantaneous.**
 
 - Localizations per frame: this will plot the localization per frame. This is useful to inspect the performance of an experiment over time.
 - Histogram: creates a histogram for the population. This is useful e.g., for comparing background signals.
@@ -88,11 +88,28 @@ Creating the plots might, therefore, not be instantaneous. **
 
 Watcher
 ~~~~~~~
-Set up a watcher to automatically process files in a folder with pre-defined settings.
+- Set up a file watcher to process files in a folder with pre-defined settings automatically.
+- All raw files and new files that haven't been processed will be processed.
+- You can define different parameter groups so that a file will be processed with different settings.
+- You can also chain custom commands to the watcher.
+- The watcher will check for the following filetypes: ('.raw', '.ome.tif', '.ims')
+
+Logging
+"""""""
+Each watcher will write their statust to a logfile. The path is visible when setting up the logfile and when checking the currently running watchers. To view the log, select the logfile and expand the `Log`-field.
+
+Multiple Parameter Groups
+"""""""""""""""""""""""""
+.. image:: ../docs/server_watcher_pg.png
+   :scale: 10 %
+   :alt: Histogram
+
+With multiple parameter groups a file will be processed with different settings. To make multiple parameter groups available, set the `Number of Parameter Groups` to a value larger than 1. 
+The file ending will be different according to the parameter group used. E.g. if you have two parameter groups, file `testfile.raw` will be processed as `testfile_pg_1_locs.hdf5` and `testfile_pg_2_locs.hdf5`.
 
-* All raw files in the folder that haven't been processed will be processed.
-* Use different folders to process files with different settings.
 
+Custom commands
+"""""""""""""""
 You can use the "Custom command" to execute a custom script after a file was processed.
 Consider the following example for a script that you want to execute named test.py:::
 
@@ -115,7 +132,6 @@ When having an existing Picasso one-click installation, Picasso can directly be
 * ``picasso link $FILENAME`` for linking localizations
 * ``picasso dbscan $FILENAME 0.1 2`` for performing DBSCAN cluster analysis with 0.1 / 2
 
-etc.
 
 Preview
 ~~~~~~~

picasso/__main__.py

@@ -916,6 +916,9 @@ def prompt_info():
                 "Min. Net Gradient": min_net_gradient,
                 "Convergence Criterion": convergence,
                 "Max. Iterations": max_iterations,
+                "Pixelsize": args.pixelsize,
+                "Fit method": args.fit_method
+
             }
 
             if args.fit_method == "lq-3d" or args.fit_method == "lq-gpu-3d":
@@ -938,7 +941,8 @@ def prompt_info():
             info.append(localize_info)
 
             base, ext = splitext(path)
-            out_path = base + "_locs.hdf5"
+
+            out_path = f"{base}{args.suffix}_locs.hdf5"
             save_locs(out_path, locs, info)
             print("File saved to {}".format(out_path))
 
@@ -950,7 +954,7 @@ def prompt_info():
             if CHECK_DB:
                 print("\n")
                 print("Assesing quality and adding to DB")
-                add_file_to_db(path)
+                add_file_to_db(path, out_path)
                 print("Done.")
                 print("\n")
 
@@ -1366,13 +1370,25 @@ def main():
     localize_parser.add_argument(
         "-mf", "--mf", type=float, default=0, help="Magnification factor (only 3d)"
     )
+    localize_parser.add_argument(
+        "-px", "--pixelsize", type=int, default=130, help="pixelsize in nm"
+    )
     localize_parser.add_argument(
         "-zc",
         "--zc",
         type=str,
         default="",
         help="Path to 3d calibration file (only 3d)",
     )
+
+    localize_parser.add_argument(
+        "-sf",
+        "--suffix",
+        type=str,
+        default="",
+        help="Suffix to add to files",
+    )
+
     localize_parser.add_argument(
         "-db",
         "--database",

picasso/gui/localize.py

@@ -1675,6 +1675,7 @@ def save_locs(self, path):
         localize_info = self.last_identification_info.copy()
         localize_info["Generated by"] = "Picasso Localize"
         localize_info["Pixelsize"] = self.parameters_dialog.pixelsize.value()
+        localize_info["Fit method"] = self.parameters_dialog.fit_method.currentText()
         if self.parameters_dialog.fit_z_checkbox.isChecked():
             localize_info[
                 "Z Calibration Path"
@@ -1934,7 +1935,7 @@ def drift_callback(x):
         print(f"Quality {nena_px} {drift_x} {drift_y} {len_mean}")
 
         localize.add_file_to_db(
-            self.path, drift=(drift_x, drift_y), len_mean=len_mean, nena=nena_px
+            self.path, None, drift=(drift_x, drift_y), len_mean=len_mean, nena=nena_px
         )
 
         self.finished.emit("Quality parameters complete.")

picasso/localize.py

@@ -388,10 +388,12 @@ def check_drift(locs, info, callback=None):
     return (drift_x, drift_y)
 
 
-def get_file_summary(file, drift=None, len_mean=None, nena=None):
+def get_file_summary(file, file_hdf, drift=None, len_mean=None, nena=None):
 
-    base, ext = os.path.splitext(file)
-    file_hdf = base + "_locs.hdf5"
+    if file_hdf is None:
+        base, ext = os.path.splitext(file)
+        file_hdf = base + "_locs.hdf5"
+
     locs, info = _io.load_locs(file_hdf)
 
     summary = {}
@@ -440,6 +442,7 @@ def get_file_summary(file, drift=None, len_mean=None, nena=None):
     summary["nena_nm"] = summary["nena_px"] * summary["pixelsize"]
 
     summary["filename"] = file
+    summary["filename_hdf"] = file_hdf
     summary["file_created"] = datetime.fromtimestamp(os.path.getmtime(file))
     summary["entry_created"] = datetime.now()
 
@@ -450,7 +453,7 @@ def _db_filename():
     home = os.path.expanduser("~")
     picasso_dir = os.path.join(home, ".picasso")
     os.makedirs(picasso_dir, exist_ok=True)
-    return os.path.abspath(os.path.join(picasso_dir, "app_044.db"))
+    return os.path.abspath(os.path.join(picasso_dir, "app_048.db"))
 
 
 def save_file_summary(summary):
@@ -459,9 +462,6 @@ def save_file_summary(summary):
     s.to_sql("files", con=engine, if_exists="append", index=False)
 
 
-def add_file_to_db(file, drift=None, len_mean=None, nena=None):
-    base, ext = os.path.splitext(file)
-    out_path = base + "_locs.hdf5"
-
-    summary = get_file_summary(file, drift, len_mean, nena)
+def add_file_to_db(file, file_hdf, drift=None, len_mean=None, nena=None):
+    summary = get_file_summary(file, file_hdf, drift, len_mean, nena)
     save_file_summary(summary)

picasso/server/compare.py

@@ -13,7 +13,7 @@
 
 
 @st.cache
-def load_file(path: str, file: str):
+def load_file(path: str):
     """Loads a localization file and returns as pandas Dataframe.
     Adds a column with the filename.
 
@@ -24,7 +24,7 @@ def load_file(path: str, file: str):
 
     locs, info = io.load_locs(path)
     locs = pd.DataFrame(locs)
-    locs["file"] = file
+    locs["file"] = os.path.split(path)[-1]
     return locs, info
 
 
@@ -170,18 +170,21 @@ def compare():
             for file in selected:
                 try:
                     c1, f1 = get_file_family(file)
-                    file_dict[file] = st.selectbox(
-                        f"Select hdf file for {file}", [None] + c1
+                    file_dict[file] = st.multiselect(
+                        f"Select hdf file for {file}",
+                        c1,
+                        None,
                     )
 
                     if file_dict[file] is not None:
-                        path = os.path.dirname(file)
+                        for _ in file_dict[file]:
+                            path = os.path.dirname(file)
 
-                        with st.spinner("Loading files"):
-                            locs, info = load_file(
-                                os.path.join(path, file_dict[file]), file_dict[file]
-                            )
-                            hdf_dict[file] = locs
+                            locs_filename = os.path.join(path, _)
+
+                            with st.spinner("Loading files"):
+                                locs, info = load_file(locs_filename)
+                                hdf_dict[locs_filename] = locs
                 except FileNotFoundError:
                     st.error(
                         f"File **{file}** was not found. Please check that this file exists."

picasso/server/helper.py

@@ -35,7 +35,8 @@ def fetch_watcher():
     try:
         engine = create_engine("sqlite:///" + _db_filename(), echo=False)
         df = pd.read_sql_table("watcher", con=engine)
-    except ValueError:
+    except ValueError as e:
+        print(e)
         df = pd.DataFrame()
 
     return df

picasso/server/history.py

@@ -216,9 +216,16 @@ def history():
             with st.spinner("Creating plots.."):
 
                 if plotmode == "Table":
-                    table = df[fields + ["filename", "file_created"]]
+                    table = df
+
+                    if st.checkbox("Barchart in column"):
+                        table_style = table.style.bar(color="lightgray")
+                        st.write(
+                            table_style.to_html(escape=False), unsafe_allow_html=True
+                        )
+                    else:
+                        st.dataframe(table)
 
-                    st.write(table.style.bar(color="gray").format(precision=4))
                     csv = convert_df(table)
 
                     st.download_button(
@@ -239,7 +246,7 @@ def history():
                                 x="file_created_",
                                 y=field,
                                 color="group",
-                                hover_name="filename",
+                                hover_name="filename_hdf",
                                 hover_data=["file_created"],
                                 title=f"{field} - median {median_:.2f}",
                                 trendline=trendline,
@@ -260,7 +267,7 @@ def history():
                                 x="file_created_date",
                                 y=field,
                                 color="group",
-                                hover_name="filename",
+                                hover_name="filename_hdf",
                                 hover_data=["file_created"],
                                 title=f"{field} - median {median_:.2f}",
                                 height=400,