From 3c34338794a7d34f92dd5311363d592c5f556307 Mon Sep 17 00:00:00 2001 From: Joran Angevaare Date: Sat, 31 Oct 2020 00:15:57 +0100 Subject: [PATCH 1/3] ignore checks for savenever plugins --- strax/plugin.py | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/strax/plugin.py b/strax/plugin.py index c25460269..03eb4c414 100644 --- a/strax/plugin.py +++ b/strax/plugin.py @@ -10,7 +10,7 @@ import logging import time import typing - +from warnings import warn from immutabledict import immutabledict import numpy as np @@ -335,13 +335,21 @@ def iter(self, iters, executor=None): self.input_buffer[d].split( t=this_chunk_end, allow_early_split=True) - # If any of the inputs were trimmed due to early splits, # trim the others too. # In very hairy cases this can take multiple passes. # TODO: can we optimize this, or code it more elegantly? max_passes_left = 10 while max_passes_left > 0: + if int(self.save_when) == strax.SaveWhen.NEVER: + # So this is a bit funny, this is the only time we may + # be actually running a plugin that is not saved with + # the time range. This means that we might be asking + # for a time range that is inconsistent with the + # chunking of the inputs. See: + # https://github.com/AxFoundation/strax/issues/247 + break + this_chunk_end = min([x.end for x in inputs.values()] + [this_chunk_end]) if len(set([x.end for x in inputs.values()])) <= 1: @@ -399,8 +407,10 @@ def cleanup(self, # Check the input buffer is empty for d, buffer in self.input_buffer.items(): if buffer is not None and len(buffer): - raise RuntimeError( - f"Plugin {d} terminated with leftover {d}: {buffer}") + # This can happen especially in time range selections + if int(self.save_when) != strax.SaveWhen.NEVER: + warn(f"Plugin {d} terminated with leftover {d}: {buffer}", + RuntimeWarning) def _check_dtype(self, x, d=None): # There is an additional 'last resort' data type check @@ -442,10 +452,27 @@ def do_compute(self, chunk_i=None, **kwargs): if len(kwargs): # Check inputs describe the same time range tranges = {k: (v.start, v.end) for k, v in kwargs.items()} - if len(set(tranges.values())) != 1: - raise ValueError(f"{self.__class__.__name__} got inconsistent " - f"time ranges of inputs: {tranges}") start, end = list(tranges.values())[0] + + # For non-saving plugins, don't be strict, just take whatever + # endtimes are available and don't check time-consistency + if int(self.save_when) != strax.SaveWhen.NEVER: + # This warning/check will be deleted, see UserWarning + if len(set(tranges.values())) != 1: + end = max([v.end for v in kwargs.values()]) # Don't delete + message = ( + f"New feature, we are ignoring inconsistent the " + f"possible ValueError in time ranges for " + f"{self.__class__.__name__} of inputs: {tranges}" + f"because this occurred in a save_when.NEVER " + f"plugin. Report any findings in " + f"github.com/AxFoundation/strax/issues/247") + warn(message, UserWarning) + # This block will be deleted + elif len(set(tranges.values())) != 1: + message = (f"{self.__class__.__name__} got inconsistent time " + f"ranges of inputs: {tranges}") + raise ValueError(message) else: # This plugin starts from scratch start, end = None, None From 92e51b643c5e6bb290f38d826ba7f137b64a5329 Mon Sep 17 00:00:00 2001 From: Joran Angevaare Date: Mon, 30 Nov 2020 15:53:01 +0100 Subject: [PATCH 2/3] remove superflous check --- strax/plugin.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/strax/plugin.py b/strax/plugin.py index 8bf45d051..061be8675 100644 --- a/strax/plugin.py +++ b/strax/plugin.py @@ -353,15 +353,6 @@ def iter(self, iters, executor=None): # TODO: can we optimize this, or code it more elegantly? max_passes_left = 10 while max_passes_left > 0: - if int(self.save_when) == strax.SaveWhen.NEVER: - # So this is a bit funny, this is the only time we may - # be actually running a plugin that is not saved with - # the time range. This means that we might be asking - # for a time range that is inconsistent with the - # chunking of the inputs. See: - # https://github.com/AxFoundation/strax/issues/247 - break - this_chunk_end = min([x.end for x in inputs.values()] + [this_chunk_end]) if len(set([x.end for x in inputs.values()])) <= 1: @@ -468,7 +459,7 @@ def do_compute(self, chunk_i=None, **kwargs): # For non-saving plugins, don't be strict, just take whatever # endtimes are available and don't check time-consistency - if int(self.save_when) != strax.SaveWhen.NEVER: + if int(self.save_when) == strax.SaveWhen.NEVER: # This warning/check will be deleted, see UserWarning if len(set(tranges.values())) != 1: end = max([v.end for v in kwargs.values()]) # Don't delete From 35b9507428122c8ec2f3c7ebcf4562f271fb33af Mon Sep 17 00:00:00 2001 From: Joran Angevaare Date: Tue, 1 Dec 2020 10:30:24 +0100 Subject: [PATCH 3/3] reinstate runtimeerror for plugins whereto this PR does not apply --- strax/plugin.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/strax/plugin.py b/strax/plugin.py index 061be8675..1956d290b 100644 --- a/strax/plugin.py +++ b/strax/plugin.py @@ -407,13 +407,13 @@ def cleanup(self, raise RuntimeError( f"Plugin {d} terminated without fetching last {d}!") - # Check the input buffer is empty - for d, buffer in self.input_buffer.items(): - if buffer is not None and len(buffer): - # This can happen especially in time range selections - if int(self.save_when) != strax.SaveWhen.NEVER: - warn(f"Plugin {d} terminated with leftover {d}: {buffer}", - RuntimeWarning) + # This can happen especially in time range selections + if int(self.save_when) != strax.SaveWhen.NEVER: + for d, buffer in self.input_buffer.items(): + # Check the input buffer is empty + if buffer is not None and len(buffer): + raise RuntimeError( + f"Plugin {d} terminated with leftover {d}: {buffer}") def _check_dtype(self, x, d=None): # There is an additional 'last resort' data type check