Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Choim nse9988 #215

Closed
wants to merge 3 commits into from
Closed

Choim nse9988 #215

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fd9b93b
add more info to err_datum logging
choim-sci Oct 4, 2023
6bf784f
Merge branch 'master' of https://github.com/NEONScience/NEON-IS-data-…
choim-sci Oct 10, 2023
ca7954a
add error routing to padded_timeseries_analyzer
choim-sci Oct 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions modules/common/err_datum.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ def err_datum_path(err: str,DirDatm: Path,DirErrBase: Path,RmvDatmOut: bool,DirO

caller = inspect.stack()[1].function
log.info(f'The error, "{err}", resulted from call: {caller}. ')
log.info(f'Rerouting starts..... ')
log.info(f'Rerouting to {DirErrBase} starts..... ')

# call to parse input directory
DirInInfo = get_dir_info(DirIn = DirDatm)
Expand All @@ -34,7 +34,7 @@ def err_datum_path(err: str,DirDatm: Path,DirErrBase: Path,RmvDatmOut: bool,DirO
DirOutBase = Path(DirErrBase).parents[0]
#
if DirInInfo == []:
log.info(f'Re-routing stopped due to the input path structure not compliant.........\n')
log.info(f'Re-routing to {DirErrBase} stopped due to the input path structure not compliant.........\n')

else:
# Inform the user of the error routing
Expand All @@ -57,3 +57,4 @@ def err_datum_path(err: str,DirDatm: Path,DirErrBase: Path,RmvDatmOut: bool,DirO
if os.path.exists(DirOut_path):
removed = shutil.rmtree(DirOut_path)
log.info(f'Removed partial output for errored datum: {DirOut_path}')

15 changes: 12 additions & 3 deletions modules/padded_timeseries_analyzer/padded_timeseries_analyzer/padded_timeseries_analyzer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from structlog import get_logger

from padded_timeseries_analyzer.padded_timeseries_analyzer.analyzer_config import AnalyzerConfig
from common.err_datum import err_datum_path

log = get_logger()

Expand All @@ -16,19 +17,24 @@ def __init__(self, data_path: Path, out_path: Path, relative_path_index: int) ->
"""
Constructor.

:param data_path: The data directory.
:param out_path: The output directory.
:param data_path: The data directory, i.e., /tmp/in/prt/2018/01/03.
:param out_path: The output directory, i.e., /tmp/out.
:param relative_path_index: Trim input file paths to this index.
"""
self.data_path = data_path
self.out_path = out_path
self.relative_path_index = relative_path_index
# DirErrBase: the user specified error directory, i.e., /tmp/out/errored
self.DirErrBase = Path(self.out_path, 'errored')

def analyze(self) -> None:
"""Verify all necessary data files are present in the input."""
manifest_file = AnalyzerConfig.manifest_filename
try:
for root, directories, files in os.walk(str(self.data_path)):
if (manifest_file not in files):
log.fatal(f'No manifest_file found in directory {self.data_path}')
sys.exit("No manifest_file found in data_path directory")
for filename in files:
if filename == manifest_file:
# read manifest
Expand Down Expand Up @@ -60,9 +66,12 @@ def analyze(self) -> None:
self.link_thresholds(file_path, link_path)
# go up one directory to find any ancillary files to link
self.link_ancillary_files(Path(root))
except Exception:
except:
exception_type, exception_obj, exception_tb = sys.exc_info()
err_msg = sys.exc_info()
log.error("Exception at line " + str(exception_tb.tb_lineno) + ": " + str(sys.exc_info()))
err_datum_path(err=err_msg,DirDatm=str(self.data_path),DirErrBase=self.DirErrBase,RmvDatmOut=True,
DirOutBase=self.out_path)

@staticmethod
def link_thresholds(data_file_path: Path, data_file_link_path: Path) -> None:
Expand Down