From 1273d22c7b51e4ae6fab6d5ee9f4737b3b231730 Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud <65791736+ahmedabu98@users.noreply.github.com> Date: Tue, 25 Jul 2023 20:08:06 +0000 Subject: [PATCH] [Python] get options outside of DoFn context in bigquery(#27662) --- sdks/python/apache_beam/io/gcp/bigquery_read_internal.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/io/gcp/bigquery_read_internal.py b/sdks/python/apache_beam/io/gcp/bigquery_read_internal.py index 6841de1e26fb..5a59b4098830 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_read_internal.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_read_internal.py @@ -144,6 +144,8 @@ def __init__(self, side_input=None): self.side_input = side_input def expand(self, input): + pipeline_options = input.pipeline.options + class PassThrough(beam.DoFn): def process(self, element): yield element @@ -151,7 +153,7 @@ def process(self, element): class CleanUpProjects(beam.DoFn): def process(self, unused_element, unused_signal, pipeline_details): bq = bigquery_tools.BigQueryWrapper.from_pipeline_options( - input.pipeline.options) + pipeline_options) pipeline_details = pipeline_details[0] if 'temp_table_ref' in pipeline_details.keys(): temp_table_ref = pipeline_details['temp_table_ref']