Skip to content

Commit

Permalink
fix(ingestion): ensure source/sink reports are always logged (#4592)
Browse files Browse the repository at this point in the history
  • Loading branch information
anshbansal authored Apr 12, 2022
1 parent 7b14871 commit 23ece3b
Showing 1 changed file with 15 additions and 5 deletions.
20 changes: 15 additions & 5 deletions metadata-ingestion/src/datahub/cli/ingest_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,11 +102,21 @@ def run(
raise SensitiveError() from e

logger.info("Starting metadata ingestion")
pipeline.run()
logger.info("Finished metadata ingestion")
ret = pipeline.pretty_print_summary(warnings_as_failure=strict_warnings)
pipeline.log_ingestion_stats()
sys.exit(ret)
try:
pipeline.run()
except Exception as e:
logger.info(
f"Source ({pipeline.config.source.type}) report:\n{pipeline.source.get_report().as_string()}"
)
logger.info(
f"Sink ({pipeline.config.sink.type}) report:\n{pipeline.sink.get_report().as_string()}"
)
raise e
else:
logger.info("Finished metadata pipeline")
pipeline.log_ingestion_stats()
ret = pipeline.pretty_print_summary(warnings_as_failure=strict_warnings)
sys.exit(ret)


def get_runs_url(gms_host: str) -> str:
Expand Down

0 comments on commit 23ece3b

Please sign in to comment.