From 8019d86be01c7145ea00dd95c2a83184cbb1446c Mon Sep 17 00:00:00 2001 From: Lukas Mikelionis Date: Tue, 27 Aug 2024 16:34:40 +0200 Subject: [PATCH] update pattern for dataflow job id extraction --- airflow/providers/google/cloud/hooks/dataflow.py | 2 +- tests/providers/google/cloud/hooks/test_dataflow.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/airflow/providers/google/cloud/hooks/dataflow.py b/airflow/providers/google/cloud/hooks/dataflow.py index 7f684cb61740f..0cc8af4527c6e 100644 --- a/airflow/providers/google/cloud/hooks/dataflow.py +++ b/airflow/providers/google/cloud/hooks/dataflow.py @@ -71,7 +71,7 @@ JOB_ID_PATTERN = re.compile( - r"Submitted job: (?P[^\"\n]*)|Created job with id: \[(?P[^\"\n]*)\]" + r"Submitted job: (?P[^\"\n\s]*)|Created job with id: \[(?P[^\"\n\s]*)\]" ) T = TypeVar("T", bound=Callable) diff --git a/tests/providers/google/cloud/hooks/test_dataflow.py b/tests/providers/google/cloud/hooks/test_dataflow.py index 39251362880db..e3a5cb1af154c 100644 --- a/tests/providers/google/cloud/hooks/test_dataflow.py +++ b/tests/providers/google/cloud/hooks/test_dataflow.py @@ -2091,6 +2091,18 @@ def test_delete_data_pipeline(self, mock_connection): > gcloud dataflow jobs --project=XXX cancel --region=europe-west3 {TEST_JOB_ID} """ +# using logback pattern: [%d] %-5level %msg \(%c\) \(%t\)%n +APACHE_BEAM_V_2_58_1_JAVA_SDK_LOG = f""""\ +[2024-08-28 08:06:39,298] INFO Dataflow SDK version: 2.58.1 (org.apache.beam.runners.dataflow.DataflowRunner) (main) +[2024-08-28 08:06:40,305] INFO To access the Dataflow monitoring console, please navigate to\ +https://console.cloud.google.com/dataflow/jobs/europe-west1/{TEST_JOB_ID}?project=XXXX\ +(org.apache.beam.runners.dataflow.DataflowRunner) (main) +[2024-08-28 08:06:40,305] INFO Submitted job: {TEST_JOB_ID} (org.apache.beam.runners.dataflow.DataflowRunner) (main) +[2024-08-28 08:06:40,305] INFO To cancel the job using the 'gcloud' tool, run: +> gcloud dataflow jobs --project=gowish-develop cancel --region=europe-west1 {TEST_JOB_ID}\ +(org.apache.beam.runners.dataflow.DataflowRunner) (main) +""" + CLOUD_COMPOSER_CLOUD_LOGGING_APACHE_BEAM_V_2_56_0_JAVA_SDK_LOG = f"""\ WARNING - {{"message":"org.apache.beam.runners.dataflow.DataflowRunner - Dataflow SDK version: 2.56.0",\ "severity":"INFO"}} @@ -2153,6 +2165,7 @@ class TestDataflow: [ pytest.param(APACHE_BEAM_V_2_14_0_JAVA_SDK_LOG, id="apache-beam-2.14.0-JDK"), pytest.param(APACHE_BEAM_V_2_22_0_JAVA_SDK_LOG, id="apache-beam-2.22.0-JDK"), + pytest.param(APACHE_BEAM_V_2_58_1_JAVA_SDK_LOG, id="apache-beam-2.58.1-JDK"), pytest.param( CLOUD_COMPOSER_CLOUD_LOGGING_APACHE_BEAM_V_2_56_0_JAVA_SDK_LOG, id="cloud-composer-cloud-logging-apache-beam-2.56.0-JDK",