From 2e030c02fccee93d45d44d50e95aea8a744093ba Mon Sep 17 00:00:00 2001 From: Damon Douglas Date: Wed, 4 Dec 2024 16:28:17 -0800 Subject: [PATCH 1/3] Move order of write_batch_size --- .github/trigger_files/beam_PostCommit_Python.json | 2 +- sdks/python/apache_beam/io/jdbc.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/trigger_files/beam_PostCommit_Python.json b/.github/trigger_files/beam_PostCommit_Python.json index 2d7af65a3815..00bd9e035648 100644 --- a/.github/trigger_files/beam_PostCommit_Python.json +++ b/.github/trigger_files/beam_PostCommit_Python.json @@ -1,5 +1,5 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run.", - "modification": 5 + "modification": 6 } diff --git a/sdks/python/apache_beam/io/jdbc.py b/sdks/python/apache_beam/io/jdbc.py index 11570680a2f3..a2237ea52354 100644 --- a/sdks/python/apache_beam/io/jdbc.py +++ b/sdks/python/apache_beam/io/jdbc.py @@ -128,11 +128,11 @@ def default_io_expansion_service(classpath=None): ('disable_autocommit', typing.Optional[bool]), ('output_parallelization', typing.Optional[bool]), ('autosharding', typing.Optional[bool]), + ('write_batch_size', typing.Optional[np.int64]), ('partition_column', typing.Optional[str]), ('partitions', typing.Optional[np.int16]), ('max_connections', typing.Optional[np.int16]), - ('driver_jars', typing.Optional[str]), - ('write_batch_size', typing.Optional[np.int64])], + ('driver_jars', typing.Optional[str])], ) DEFAULT_JDBC_CLASSPATH = ['org.postgresql:postgresql:42.2.16'] From 6b3c7ee0722e6e7b7b453047fe9a333428189a88 Mon Sep 17 00:00:00 2001 From: Damon Douglas Date: Thu, 5 Dec 2024 10:14:11 -0800 Subject: [PATCH 2/3] Reorder Java schema instead --- .../apache/beam/sdk/io/jdbc/JdbcSchemaIOProvider.java | 2 +- sdks/python/apache_beam/io/jdbc.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcSchemaIOProvider.java b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcSchemaIOProvider.java index 11034aee1cdf..23221042938b 100644 --- a/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcSchemaIOProvider.java +++ b/sdks/java/io/jdbc/src/main/java/org/apache/beam/sdk/io/jdbc/JdbcSchemaIOProvider.java @@ -68,13 +68,13 @@ public Schema configurationSchema() { .addNullableField("disableAutoCommit", FieldType.BOOLEAN) .addNullableField("outputParallelization", FieldType.BOOLEAN) .addNullableField("autosharding", FieldType.BOOLEAN) - .addNullableField("writeBatchSize", FieldType.INT64) // Partitioning support. If you specify a partition column we will use that instead of // readQuery .addNullableField("partitionColumn", FieldType.STRING) .addNullableField("partitions", FieldType.INT16) .addNullableField("maxConnections", FieldType.INT16) .addNullableField("driverJars", FieldType.STRING) + .addNullableField("writeBatchSize", FieldType.INT64) .build(); } diff --git a/sdks/python/apache_beam/io/jdbc.py b/sdks/python/apache_beam/io/jdbc.py index a2237ea52354..fd446d452629 100644 --- a/sdks/python/apache_beam/io/jdbc.py +++ b/sdks/python/apache_beam/io/jdbc.py @@ -119,8 +119,11 @@ def default_io_expansion_service(classpath=None): Config = typing.NamedTuple( 'Config', - [('driver_class_name', str), ('jdbc_url', str), ('username', str), - ('password', str), ('connection_properties', typing.Optional[str]), + [('driver_class_name', str), + ('jdbc_url', str), + ('username', str), + ('password', str), + ('connection_properties', typing.Optional[str]), ('connection_init_sqls', typing.Optional[typing.List[str]]), ('read_query', typing.Optional[str]), ('write_statement', typing.Optional[str]), @@ -128,11 +131,11 @@ def default_io_expansion_service(classpath=None): ('disable_autocommit', typing.Optional[bool]), ('output_parallelization', typing.Optional[bool]), ('autosharding', typing.Optional[bool]), - ('write_batch_size', typing.Optional[np.int64]), ('partition_column', typing.Optional[str]), ('partitions', typing.Optional[np.int16]), ('max_connections', typing.Optional[np.int16]), - ('driver_jars', typing.Optional[str])], + ('driver_jars', typing.Optional[str]), + ('write_batch_size', typing.Optional[np.int64])], ) DEFAULT_JDBC_CLASSPATH = ['org.postgresql:postgresql:42.2.16'] From 5896f1dc18fa557f281588be7a0db02be64a09ef Mon Sep 17 00:00:00 2001 From: Damon Douglas Date: Thu, 5 Dec 2024 10:26:02 -0800 Subject: [PATCH 3/3] Revert jdbc.py changes --- sdks/python/apache_beam/io/jdbc.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/sdks/python/apache_beam/io/jdbc.py b/sdks/python/apache_beam/io/jdbc.py index fd446d452629..11570680a2f3 100644 --- a/sdks/python/apache_beam/io/jdbc.py +++ b/sdks/python/apache_beam/io/jdbc.py @@ -119,11 +119,8 @@ def default_io_expansion_service(classpath=None): Config = typing.NamedTuple( 'Config', - [('driver_class_name', str), - ('jdbc_url', str), - ('username', str), - ('password', str), - ('connection_properties', typing.Optional[str]), + [('driver_class_name', str), ('jdbc_url', str), ('username', str), + ('password', str), ('connection_properties', typing.Optional[str]), ('connection_init_sqls', typing.Optional[typing.List[str]]), ('read_query', typing.Optional[str]), ('write_statement', typing.Optional[str]),