Update from dev

CHANGELOG.md

@@ -6,9 +6,11 @@ All notable changes to this project will be documented in this file.
 ### NEW
 
 ### CHANGED
+- major performance improvements for DiscreteData #403 #418, #424
 
 ### Fixed
 - fix bug #394 'Copying a spy.StructDict returns a dict'.
+- serializable `.cfg` #392
 
 ## [2022.12]
 

doc/source/developer/developers.rst

@@ -3,7 +3,7 @@ Syncopy Developer Guide
 ***********************
 
 The following information is meant for advanced users with an understanding of
-class hierarchies that want to extend and/or modify Syncopy's base functionality. 
+class hierarchies that want to extend and/or modify Syncopy's base functionality.
 
 .. toctree::
     :glob:
@@ -13,4 +13,5 @@ class hierarchies that want to extend and/or modify Syncopy's base functionality
     io
     tools
     compute_kernels
+    logging
     developer_api

doc/source/developer/logging.rst

@@ -0,0 +1,75 @@
+.. _syncopy-logging:
+
+Controlling Logging in Syncopy
+===============================
+
+Syncopy uses the `Python logging module <https://docs.python.org/3/library/logging.html>`_ for logging. It uses two different loggers:
+one for code that runs on the local machine, and another one for logging the parallelelized code that
+is run by the remote workers in a high performance computing (HPC) cluster environment.
+
+
+Log levels
+-----------
+
+The default log level is for the Syncopy logger is `'logging.IMPORTANT'` (from now on referred to as `'IMPORTANT'`). This means that you will not see any Syncopy messages below that threshold, i.e., messages printed with log levels `'DEBUG'` and `'INFO'`. To change the log level, you can either use the logging API in your application code as explained below, or set the environment variable `'SPYLOGLEVEL'` to one of the values supported by the logging module, e.g., 'CRITICAL', 'WARNING', 'INFO', or 'DEBUG'. See the `official docs of the logging module <https://docs.python.org/3/library/logging.html#levels>`_ for details on the supported log levels. Note that IMPORTANT is a custom log level with importance 25, i.e., between INFO and WARNING.
+
+
+Log file location
+-----------------
+
+All Syncopy log files are saved in a configurable directory which we refer to as `SPYLOGDIR`. By default, `SPYLOGDIR` is set to the directory `.spy/logs/` in your home directory (accessible as `~/.spy/logs/` under Linux and Mac OS), and it can be adapted by setting the environment variable `SPYLOGDIR` before running your application.
+
+E.g., if your Python script using Syncopy is `~/neuro/paperfig1.py`, you can set the log level and log directory on the command line like this in the Bash shell:
+
+.. code-block:: shell
+   export SPYLOGDIR=/tmp/spy
+   export SPYLOGLEVEL=DEBUG
+   ~/neuro/paperfig1.py
+
+
+Logging code that runs locally
+-------------------------------
+
+For all code that is run on the local machine, Syncopy logs to a logger named `'syncopy'` which is handled by both the console and the logfile `'SPYLOGDIR/syncopy.log'`.
+
+To adapt the local logging behaviour of Syncopy, one can configure the logger as explained in the documentation for the logging module, e.g., in your application that uses Syncopy:
+
+.. code-block:: python
+
+   import syncopy
+   import logging
+   # Get the logger used by syncopy
+   logger = logging.getLogger('syncopy')
+
+   # Change the log level:
+   logger.setLevel(logging.DEBUG)
+
+   # Add another handler that logs to a file:
+   fh = logging.FileHandler('syncopy_debug_log.log')
+   logger.addHandler(fh)
+
+   logger.info("My app starts now.")
+   # The rest of your application code goes here.
+
+
+Logging code that potentially runs remotely
+--------------------------------------------
+
+The parallel code that performs the heavy lifting on the Syncopy data (i.e., what we call `compute functions`) will be executed on remote machines when Syncopy is run in an HPC environment. Therefore,
+special handling is required for these parts of the code, and we need to log to one log file per remote machine.
+
+Syncopy automatically configures a suitable logger named `syncopy_<host>` on each host, where `<host>` is the hostname. Each of these loggers is attached to the respective logfile `'SPYLOGDIR/syncopy_<host>.log'`, where `<host>` is the hostname, which ensures that logging works properly even if you log into the same directory on all remote machines (e.g., a home directory that is mounted on all machines via a network file system).
+
+Here is how to log with the remote logger:
+
+.. code-block:: python
+
+   import syncopy
+   import logging, platform
+
+   # ...
+   # In some cF or backend function:
+   par_logger = logging.getLogger("syncopy_" + platform.node())
+   par_logger.info("Code run on remote machine is being run.")
+
+This is all you need to do. If you want to configure different log levels for the remote logger and the local one, you can configure the environment variable `SPYPARLOGLEVEL` in addition to `SPYLOGLEVEL`.

syncopy/__init__.py

@@ -7,8 +7,8 @@
 import os
 import sys
 import subprocess
-import socket
 import getpass
+import socket
 import numpy as np
 from hashlib import blake2b, sha1
 from importlib.metadata import version, PackageNotFoundError
@@ -36,17 +36,23 @@
 
 # --- Greeting ---
 
+def startup_print_once(message):
+    """Print message once: do not spam message n times during all n worker imports."""
+    try:
+        dd.get_client()
+    except ValueError:
+        silence_file = os.path.join(os.path.expanduser("~"), ".spy", "silentstartup")
+        if os.getenv("SPYSILENTSTARTUP") is None and not os.path.isfile(silence_file):
+            print(message)
+
+
 msg = f"""
 Syncopy {__version__}
 
 See https://syncopy.org for the online documentation.
 For bug reports etc. please send an email to [email protected]
 """
-# do not spam via worker imports
-try:
-    dd.get_client()
-except ValueError:
-    print(msg)
+startup_print_once(msg)
 
 # Set up sensible printing options for NumPy arrays
 np.set_printoptions(suppress=True, precision=4, linewidth=80)
@@ -90,28 +96,37 @@
 
 # Set package-wide temp directory
 csHome = "/cs/home/{}".format(getpass.getuser())
+if os.environ.get("SPYDIR"):
+    spydir = os.path.abspath(os.path.expanduser(os.environ["SPYDIR"]))
+    if not os.path.exists(spydir):
+        raise ValueError(f"Environment variable SPYDIR set to non-existent or unreadable directory '{spydir}'. Please unset SPYDIR or create the directory.")
+else:
+    if os.path.exists(csHome): # ESI cluster.
+        spydir = os.path.join(csHome, ".spy")
+    else:
+        spydir = os.path.abspath(os.path.join(os.path.expanduser("~"), ".spy"))
+
 if os.environ.get("SPYTMPDIR"):
     __storage__ = os.path.abspath(os.path.expanduser(os.environ["SPYTMPDIR"]))
 else:
-    if os.path.exists(csHome):
-        __storage__ = os.path.join(csHome, ".spy")
-    else:
-        __storage__ = os.path.join(os.path.expanduser("~"), ".spy")
+    __storage__ = os.path.join(spydir, "tmp_storage")
+
+if not os.path.exists(spydir):
+        os.makedirs(spydir, exist_ok=True)
 
 # Set upper bound for temp directory size (in GB)
 __storagelimit__ = 10
 
 # Establish ID and log-file for current session
 __sessionid__ = blake2b(digest_size=2, salt=os.urandom(blake2b.SALT_SIZE)).hexdigest()
-__sessionfile__ = os.path.join(__storage__, "session_{}.id".format(__sessionid__))
 
 # Set max. no. of lines for traceback info shown in prompt
 __tbcount__ = 5
 
 # Set checksum algorithm to be used
 __checksum_algorithm__ = sha1
 
-# Fill up namespace
+# Fill namespace
 from . import (
     shared,
     io,
@@ -126,8 +141,23 @@
 from .plotting import *
 from .preproc import *
 
-# Register session
-__session__ = datatype.base_data.SessionLogger()
+from .datatype.util import setup_storage
+storage_tmpdir_size_gb, storage_tmpdir_numfiles = setup_storage()  # Creates the storage dir if needed and computes size and number of files in there if any.
+
+from .shared.log import setup_logging
+__logdir__ = None  # Gets set in setup_logging() call below.
+setup_logging(spydir=spydir, session=__sessionid__)  # Sets __logdir__.
+startup_print_once(f"Logging to log directory '{__logdir__}'.\nTemporary storage directory set to '{__storage__}'.\n")
+
+if storage_tmpdir_size_gb > __storagelimit__:
+            msg = (
+                "\nSyncopy <core> WARNING: Temporary storage folder {tmpdir:s} "
+                + "contains {nfs:d} files taking up a total of {sze:4.2f} GB on disk. \n"
+                + "Consider running `spy.cleanup()` to free up disk space."
+            )
+            msg_formatted = msg.format(tmpdir=__storage__, nfs=storage_tmpdir_numfiles, sze=storage_tmpdir_size_gb)
+            startup_print_once(msg_formatted)
+
 
 # Override default traceback (differentiate b/w Jupyter/iPython and regular Python)
 from .shared.errors import SPYExceptionHandler

syncopy/datatype/__init__.py

@@ -13,6 +13,7 @@
 from .methods.selectdata import *
 from .methods.show import *
 from .methods.copy import *
+from .util import *
 
 # Populate local __all__ namespace
 __all__ = []
@@ -25,3 +26,4 @@
 __all__.extend(methods.selectdata.__all__)
 __all__.extend(methods.show.__all__)
 __all__.extend(methods.copy.__all__)
+__all__.extend(util.__all__)