Merge pull request #6 from AxFoundation/master

update from fork

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.7.5
+current_version = 0.8.0
 files = setup.py strax/__init__.py docs/source/conf.py
 commit = True
 tag = True

HISTORY.md

@@ -1,3 +1,11 @@
+
+0.8.0 / 2019-09-16
+------------------
+- Superruns (#207)
+- Pulse processing fixes (#207)
+- LZ4 compression (#207)
+- Fixes for edge cases (#201)
+
 0.7.5 / 2019-07-06
 ------------------
 - Time selection refactor and context extensibility (#195)

docs/source/advanced/superrun.rst

@@ -0,0 +1,53 @@
+Superruns
+=========
+
+Overview and motivation
+------------------------
+A superrun is a run defined by (parts of) other runs, which are called 'subruns'. Superrun names start with an underscore. Regular run names cannot start with an underscore.
+
+Strax builds data for a superrun by loading (and potentially building) each of the subruns, then slicing and concatenating them as necessary.
+
+Superruns are useful to track common groupings of data. For example:
+
+* 'Minimum bias' runs, consisting only of low-energy events, events passing some cuts, DM-candidates, PMT flashes, or other thing of interest. The low-level data of these is much smaller than that of all the full runs, and can be brought to a local analysis facility, enabling on-site low-level waveform watching.
+* Grouping similar runs. For example, shifters might group good runs from a week of calibration data with some source under a single name, e.g. `_kr_feb2019`.
+
+
+
+Superruns can be built from other superruns. Thus, _sr1_v0.2 could be built from _background_january, _background_february, etc.
+
+
+Defining superruns
+-------------------
+Use the `define_run` context method to define a new superrun. There are three ways to do this.
+
+From a list of runids::
+
+    st.define_run('_awesome_superrun', ['123', '124'])
+
+From a dictionary of time range tuples. The times must be 64-bit integer UTC timestamps since the unix epoch::
+
+    st.define_run('_awesome_superrun', {
+        '123': [(start, stop), (start, stop), ...],
+        '124': [(start, stop), (start, stop), ...],})
+
+From a dataframe (or record array) with strax data::
+
+    st.define_run('_awesome_superrun', events_df)
+    st.define_run('_awesome_superrun', events_df, from_run='123')
+
+In this case, the run will be made of the time ranges that correspond exactly to `events_df`. If `events_df` already has a `run_id` field (e.g. because it consists of data from multiple runs), you do not need to pass `from_run`, it will be read off from the data.
+
+It is up to the storage frontent to process your request for defining a run. As a normal user, you generally only have permissions to create a new run in the `DataDirectory` (local files) storage frontend, where runs are recorded in json files.
+
+
+How superruns work
+--------------------
+
+As mentioned above, strax builds data for superruns by slicing data of the subruns. Thus, peaks from a superrun come from the peaks of the subruns, which are built from their own records as usual.
+
+Defaults for settings can be runid-dependent in strax. If an option specifies `default_per_run=[(run, setting), (run2, setting2)]`, then runs in between run and run2 will use setting, and runs after run2 `setting2`. Superruns store a deterministic hash of this `default_per_run` specification for tracking purposes.
+
+You cannot currently go directly from the superrun's records to the superrun's peaks. This would be tricky to implement, since (1) (2) even with the same settings, many plugins choose to do something different depending on the runid. For example, in straxen the gain model is specified by a file, but which gains from the file are actually used is dependent on the runid.
+
+Thus, superruns won't help build data faster, but they will speed up loading data after it has been built. This is important, because strax' overhead for loading a run is larger than hax, due to its version and option tracking.

docs/source/conf.py

@@ -65,9 +65,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '0.7.5'
+version = '0.8.0'
 # The full version, including alpha/beta/rc tags.
-release = '0.7.5'
+release = '0.8.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/source/index.rst

@@ -31,6 +31,7 @@ You might also find these presentations useful:
     :caption: Advanced usage
 
     advanced/overview
+    advanced/superrun
     advanced/plugin_dev
 
 .. toctree::

docs/source/reference/strax.processing.rst

@@ -8,55 +8,55 @@ strax.processing.data\_reduction module
 ---------------------------------------
 
 .. automodule:: strax.processing.data_reduction
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 strax.processing.general module
 -------------------------------
 
 .. automodule:: strax.processing.general
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 strax.processing.peak\_building module
 --------------------------------------
 
 .. automodule:: strax.processing.peak_building
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 strax.processing.peak\_properties module
 ----------------------------------------
 
 .. automodule:: strax.processing.peak_properties
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 strax.processing.peak\_splitting module
 ---------------------------------------
 
 .. automodule:: strax.processing.peak_splitting
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 strax.processing.pulse\_processing module
 -----------------------------------------
 
 .. automodule:: strax.processing.pulse_processing
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 
 Module contents
 ---------------
 
 .. automodule:: strax.processing
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/reference/strax.rst

@@ -6,8 +6,8 @@ Subpackages
 
 .. toctree::
 
-    strax.processing
-    strax.storage
+   strax.processing
+   strax.storage
 
 Submodules
 ----------
@@ -16,79 +16,87 @@ strax.chunk\_arrays module
 --------------------------
 
 .. automodule:: strax.chunk_arrays
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 strax.config module
 -------------------
 
 .. automodule:: strax.config
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 strax.context module
 --------------------
 
 .. automodule:: strax.context
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 strax.dtypes module
 -------------------
 
 .. automodule:: strax.dtypes
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 strax.io module
 ---------------
 
 .. automodule:: strax.io
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 strax.mailbox module
 --------------------
 
 .. automodule:: strax.mailbox
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 strax.plugin module
 -------------------
 
 .. automodule:: strax.plugin
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 strax.processor module
 ----------------------
 
 .. automodule:: strax.processor
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+strax.run\_selection module
+---------------------------
+
+.. automodule:: strax.run_selection
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 strax.utils module
 ------------------
 
 .. automodule:: strax.utils
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 
 Module contents
 ---------------
 
 .. automodule:: strax
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/reference/strax.storage.rst

@@ -8,47 +8,47 @@ strax.storage.common module
 ---------------------------
 
 .. automodule:: strax.storage.common
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 strax.storage.files module
 --------------------------
 
 .. automodule:: strax.storage.files
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 strax.storage.mongo module
 --------------------------
 
 .. automodule:: strax.storage.mongo
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 strax.storage.s3 module
 -----------------------
 
 .. automodule:: strax.storage.s3
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 strax.storage.zipfiles module
 -----------------------------
 
 .. automodule:: strax.storage.zipfiles
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 
 Module contents
 ---------------
 
 .. automodule:: strax.storage
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   :members:
+   :undoc-members:
+   :show-inheritance:

requirements.txt

@@ -10,3 +10,4 @@ numexpr==2.6.9
 boto3==1.9.174
 scipy==1.3.0
 pymongo==3.8.0
+lz4

setup.py

@@ -12,7 +12,7 @@
     history = file.read()
 
 setuptools.setup(name='strax',
-                 version='0.7.5',
+                 version='0.8.0',
                  description='Streaming analysis for xenon TPCs',
                  author='Jelle Aalbers',
                  url='https://github.com/AxFoundation/strax',

strax/__init__.py

@@ -1,5 +1,5 @@
 # flake8: noqa
-__version__ = '0.7.5'
+__version__ = '0.8.0'
 
 # Glue the package together
 # See https://www.youtube.com/watch?v=0oTh1CXRaQ0 if this confuses you