Skip to content

Commit

Permalink
Continueing rule execution in case we get TensorUnavailable exception (
Browse files Browse the repository at this point in the history
…aws#142)

and rule might be looking for tensor which may be saved in future step.
  • Loading branch information
Vikas-kum authored Jan 10, 2020
1 parent 0c9af91 commit f5c882c
Showing 1 changed file with 7 additions and 2 deletions.
9 changes: 7 additions & 2 deletions smdebug/rules/rule_invoker.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
# First Party
from smdebug.core.logger import get_logger
from smdebug.exceptions import RuleEvaluationConditionMet, StepUnavailable, TensorUnavailableForStep
from smdebug.exceptions import (
RuleEvaluationConditionMet,
StepUnavailable,
TensorUnavailable,
TensorUnavailableForStep,
)

logger = get_logger()

Expand All @@ -11,7 +16,7 @@ def invoke_rule(rule_obj, start_step=0, end_step=None, raise_eval_cond=False):
while (end_step is None) or (step < end_step):
try:
rule_obj.invoke(step)
except (TensorUnavailableForStep, StepUnavailable) as e:
except (TensorUnavailableForStep, StepUnavailable, TensorUnavailable) as e:
logger.debug(str(e))
except RuleEvaluationConditionMet as e:
if raise_eval_cond:
Expand Down

0 comments on commit f5c882c

Please sign in to comment.