Merge branch 'master' into no-native-bq-read

.github/actions/gradle-command-self-hosted-action/action.yml

@@ -41,5 +41,9 @@ runs:
     - name: Run Gradle Command
       shell: bash
       run: |
+        # Removing settings.xml is a workaround to avoid a decryption issue
+        # of Beam's gradle-command-action plugin and github's provided
+        # maven settings.xml file
+        rm ~/.m2/settings.xml
         ./gradlew ${{ inputs.gradle-command }} --max-workers=${{ inputs.max-workers }} ${{ inputs.arguments }} \
         ${{ inputs.default-arguments }}

.github/actions/setup-self-hosted-action/action.yml

@@ -68,10 +68,3 @@ runs:
       uses: actions/setup-go@v3
       with:
         go-version: '1.18.0'
-    - name: Remove default github maven configuration
-      if: ${{ inputs.requires-java-8 == 'true'  }}
-      # This step is a workaround to avoid a decryption issue of Beam's
-      # gradle-command-action plugin and github's provided maven
-      # settings.xml file
-      shell: bash
-      run: rm ~/.m2/settings.xml

.github/workflows/cancel.yml

@@ -27,6 +27,12 @@ on:
 
 jobs:
   cancel-duplicate-workflow-runs:
+    permissions:
+      actions: write
+      checks: read
+      contents: read
+      issues: write
+      pull-requests: write
     name: "Cancel duplicate workflow runs"
     runs-on: ubuntu-latest
     steps:

.github/workflows/issue-tagger.yml

@@ -20,12 +20,12 @@ on:
 
 jobs:
   label:
-
     runs-on: ubuntu-latest
-
+    permissions:
+      issues: write
     steps:
     - uses: actions/checkout@v3
     - uses: damccorm/tag-ur-it@6fa72bbf1a2ea157b533d7e7abeafdb5855dbea5
       with:
         repo-token: "${{ secrets.GITHUB_TOKEN }}"
-        configuration-path: "./.github/issue-rules.yml"
+        configuration-path: "./.github/issue-rules.yml"

.github/workflows/pr-bot-new-prs.yml

@@ -23,6 +23,12 @@ on:
 
 jobs:
   process-prs:
+    permissions:
+      contents: write
+      pull-requests: write
+      checks: read
+      issues: read
+      statuses: read
     # Don't run on forks
     if: github.repository == 'apache/beam'
     runs-on: ubuntu-latest

.github/workflows/pr-bot-prs-needing-attention.yml

@@ -23,6 +23,12 @@ on:
 
 jobs:
   process-prs:
+    permissions:
+      contents: write
+      pull-requests: write
+      checks: read
+      issues: read
+      statuses: read
     # Don't run on forks
     if: github.repository == 'apache/beam'
     runs-on: ubuntu-latest

.github/workflows/pr-bot-update-reviewers.yml

@@ -23,6 +23,12 @@ on:
 
 jobs:
   update-reviewers:
+    permissions:
+      contents: write
+      pull-requests: write
+      checks: read
+      issues: read
+      statuses: read
     # Don't run on forks
     if: github.repository == 'apache/beam'
     runs-on: ubuntu-latest

.github/workflows/self-assign.yml

@@ -19,6 +19,8 @@ on:
 
 jobs:
   assign:
+    permissions:
+      issues: write
     name: Take or close an issue
     if: ${{ !github.event.issue.pull_request }}
     runs-on: ubuntu-latest

.github/workflows/stale.yml

@@ -28,7 +28,7 @@ jobs:
       pull-requests: write
 
     steps:
-    - uses: actions/stale@v5
+    - uses: actions/stale@v6
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-pr-message: 'This pull request has been marked as stale due to 60 days of inactivity. It will be closed in 1 week if no further activity occurs. If you think that’s incorrect or this pull request requires a review, please simply write any comment. If closed, you can revive the PR at any time and @mention a reviewer or discuss it on the [email protected] list. Thank you for your contributions.'

.github/workflows/triaged-on-assign.yml

@@ -20,6 +20,8 @@ on:
 
 jobs:
   assign:
+    permissions:
+      issues: write
     name: Mark issue as triaged when assigned
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -120,12 +120,13 @@ website/www/yarn-error.log
 **/node_modules
 
 # Dart/Flutter
-**/.dart_tool
-**/.packages
 **/.flutter-plugins
 **/.flutter-plugins-dependencies
+**/.dart_tool
 **/generated_plugin_registrant.dart
+**/*.g.dart
 **/*.mocks.dart
+**/.packages
 
 # Ignore Beam Playground Terraform
 **/.terraform

.test-infra/jenkins/job_Java_Jmh.groovy

@@ -36,7 +36,13 @@ CronJobBuilder.cronJob('beam_Java_Jmh', 'H 0 * * 0', this) {
 
     gradle {
       rootBuildScriptDir(commonJobProperties.checkoutDir)
-      tasks(':javaJmh')
+      tasks(':sdks:java:harness:jmh:jmh')
+      commonJobProperties.setGradleSwitches(delegate)
+    }
+
+    gradle {
+      rootBuildScriptDir(commonJobProperties.checkoutDir)
+      tasks(':sdks:java:core:jmh:jmh')
       commonJobProperties.setGradleSwitches(delegate)
     }
   }

.test-infra/jenkins/job_Release_NightlySnapshot.groovy

@@ -50,7 +50,7 @@ job('beam_Release_NightlySnapshot') {
     }
     /*
      * Skipping verification on 'ubuntu' labelled nodes since they don't have access to the
-     * some required GCP resouces.
+     * some required GCP resources.
      * TODO: Uncomment this after we publishing snapshots on 'beam' nodes.
      gradle {
      rootBuildScriptDir(commonJobProperties.checkoutDir)

.test-infra/metrics/grafana/dashboards/perftests_metrics/Python_ML_RunInference_Benchmark_Tests.json

@@ -1068,6 +1068,6 @@
   },
   "timezone": "",
   "title": "Python ML RunInference Benchmarks",
-  "uid": "1cnwVDkGk",
-  "version": 47
+  "uid": "ZpS8Uf44z",
+  "version": 1
 }

CHANGES.md

@@ -63,6 +63,7 @@
 * Fixed bug when using SpannerIO with RuntimeValueProvider options (Java) ([#22146](https://github.com/apache/beam/issues/22146)).
 * Fixed issue for unicode rendering on WriteToBigQuery ([#10785](https://github.com/apache/beam/issues/10785))
 * Remove obsolete variant of BigQuery Read ([#23559](https://github.com/apache/beam/issues/23559)).
+* Bumped google-cloud-spanner dependency version to 3.x for Python SDK ([#21198](https://github.com/apache/beam/issues/21198)).
 
 ## New Features / Improvements
 
@@ -79,6 +80,8 @@
 ## Bugfixes
 
 * Fixed X (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
+* Fixed Python cross-language JDBC IO Connector cannot read or write rows containing Numeric/Decimal type values ([#19817](https://github.com/apache/beam/issues/19817)).
+
 ## Known Issues
 
 * ([#X](https://github.com/apache/beam/issues/X)).

build.gradle.kts

@@ -244,11 +244,6 @@ tasks.register("javaHadoopVersionsTest") {
   dependsOn(":runners:spark:3:hadoopVersionsTest")
 }
 
-tasks.register("javaJmh"){
-  dependsOn(":sdks:java:harness:jmh:jmh")
-  dependsOn(":sdks:java:core:jmh:jmh")
-}
-
 tasks.register("sqlPostCommit") {
   dependsOn(":sdks:java:extensions:sql:postCommit")
   dependsOn(":sdks:java:extensions:sql:jdbc:postCommit")

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -464,15 +464,15 @@ class BeamModulePlugin implements Plugin<Project> {
     def classgraph_version = "4.8.104"
     def errorprone_version = "2.10.0"
     // Try to keep gax_version consistent with gax-grpc version in google_cloud_platform_libraries_bom
-    def gax_version = "2.18.7"
+    def gax_version = "2.19.2"
     def google_clients_version = "2.0.0"
     def google_cloud_bigdataoss_version = "2.2.8"
     // Try to keep google_cloud_spanner_version consistent with google_cloud_spanner_bom in google_cloud_platform_libraries_bom
-    def google_cloud_spanner_version = "6.29.0"
+    def google_cloud_spanner_version = "6.31.2"
     def google_code_gson_version = "2.9.1"
     def google_oauth_clients_version = "1.34.1"
     // Try to keep grpc_version consistent with gRPC version in google_cloud_platform_libraries_bom
-    def grpc_version = "1.48.0"
+    def grpc_version = "1.49.2"
     def guava_version = "31.1-jre"
     def hadoop_version = "2.10.2"
     def hamcrest_version = "2.1"
@@ -488,7 +488,7 @@ class BeamModulePlugin implements Plugin<Project> {
     def postgres_version = "42.2.16"
     def powermock_version = "2.0.9"
     // Try to keep protobuf_version consistent with the protobuf version in google_cloud_platform_libraries_bom
-    def protobuf_version = "3.21.4"
+    def protobuf_version = "3.21.7"
     def quickcheck_version = "1.0"
     def sbe_tool_version = "1.25.1"
     def slf4j_version = "1.7.30"
@@ -600,7 +600,7 @@ class BeamModulePlugin implements Plugin<Project> {
         // The GCP Libraries BOM dashboard shows the versions set by the BOM:
         // https://storage.googleapis.com/cloud-opensource-java-dashboard/com.google.cloud/libraries-bom/25.2.0/artifact_details.html
         // Update libraries-bom version on sdks/java/container/license_scripts/dep_urls_java.yaml
-        google_cloud_platform_libraries_bom         : "com.google.cloud:libraries-bom:26.1.1",
+        google_cloud_platform_libraries_bom         : "com.google.cloud:libraries-bom:26.1.3",
         google_cloud_spanner                        : "com.google.cloud:google-cloud-spanner", // google_cloud_platform_libraries_bom sets version
         google_cloud_spanner_test                   : "com.google.cloud:google-cloud-spanner:$google_cloud_spanner_version:tests",
         google_code_gson                            : "com.google.code.gson:gson:$google_code_gson_version",

learning/tour-of-beam/backend/internal/entity.go

@@ -33,8 +33,6 @@ type Unit struct {
 	Hints             []string `json:"hints,omitempty"`
 	TaskSnippetId     string   `json:"taskSnippetId,omitempty"`
 	SolutionSnippetId string   `json:"solutionSnippetId,omitempty"`
-	TaskName          string   `json:"-"`
-	SolutionName      string   `json:"-"`
 
 	// optional, user-specific
 	UserSnippetId string `json:"userSnippetId,omitempty"`
@@ -50,6 +48,7 @@ const (
 )
 
 type Group struct {
+	Id    string `json:"id"`
 	Title string `json:"title"`
 	Nodes []Node `json:"nodes"`
 }

learning/tour-of-beam/backend/internal/fs_content/builders.go

@@ -16,19 +16,36 @@
 package fs_content
 
 import (
+	"fmt"
+
 	tob "beam.apache.org/learning/tour-of-beam/backend/internal"
 )
 
+const OriginTbExamples = "TB_EXAMPLES"
+
+// Construct example/snippet_id from (sdk, name)
+// according to the convention defined
+// in Playground datastore
+func makeTbSnippetId(sdk tob.Sdk, name string) string {
+	// leave blank if not defined
+	if name == "" {
+		return ""
+	}
+	return fmt.Sprintf("%s_%s_%s", OriginTbExamples, sdk.StorageID(), name)
+}
+
 type UnitBuilder struct {
 	tob.Unit
 }
 
-func NewUnitBuilder(info learningUnitInfo) UnitBuilder {
+func NewUnitBuilder(info learningUnitInfo, sdk tob.Sdk) UnitBuilder {
+	taskSnippetId := makeTbSnippetId(sdk, info.TaskName)
+	solutionSnippetId := makeTbSnippetId(sdk, info.SolutionName)
 	return UnitBuilder{tob.Unit{
-		Id:           info.Id,
-		Title:        info.Name,
-		TaskName:     info.TaskName,
-		SolutionName: info.SolutionName,
+		Id:                info.Id,
+		Title:             info.Name,
+		TaskSnippetId:     taskSnippetId,
+		SolutionSnippetId: solutionSnippetId,
 	}}
 }