Merge branch 'master' into BEAM-11516

.test-infra/metrics/grafana/provisioning/datasources/beamgithubjavatests-api.yaml

@@ -21,14 +21,13 @@ apiVersion: 1
 deleteDatasources:
 
 datasources:
-  - name: Java Tests
+  - name: "Java Tests"
     type: marcusolsson-json-datasource
     access: proxy
     orgId: 1
     url: https://api.github.com/repos/apache/beam/actions/workflows/java_tests.yml/runs
     jsonData:
       httpHeaderName1: "accept"
-      customQueryParameters: "per_page=100"
     secureJsonData:
       httpHeaderValue1: "application/vnd.github.v3+json"
     editable: false

.test-infra/metrics/grafana/provisioning/datasources/beamgithubpythontests-api.yaml

@@ -21,14 +21,13 @@ apiVersion: 1
 deleteDatasources:
 
 datasources:
-  - name: Python Tests
+  - name: "Python Tests"
     type: marcusolsson-json-datasource
     access: proxy
     orgId: 1
     url: https://api.github.com/repos/apache/beam/actions/workflows/python_tests.yml/runs
     jsonData:
       httpHeaderName1: "accept"
-      customQueryParameters: "per_page=100"
     secureJsonData:
       httpHeaderValue1: "application/vnd.github.v3+json"
     editable: false

.test-infra/metrics/kubernetes/beamgrafana-deploy.yaml

@@ -46,6 +46,8 @@ spec:
           value: "true"
         - name: GF_AUTH_ANONYMOUS_ORG_NAME
           value: Beam
+        - name: GF_INSTALL_PLUGINS
+          value: marcusolsson-json-datasource
         - name: GF_SECURITY_ADMIN_PASSWORD
           valueFrom:
             secretKeyRef:
@@ -89,6 +91,7 @@ spec:
         volumeMounts:
         - mountPath: /var/lib/grafana
           name: beam-grafana-libdata
+          readOnly: false
         - mountPath: /etc/grafana
           name: beam-grafana-etcdata
         - mountPath: /var/log/grafana

CHANGES.md

@@ -61,7 +61,7 @@
 ## I/Os
 
 * Support for X source added (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)).
-* `ReadFromBigQuery` now runs queries with BATCH priority by default. The `query_priority` parameter is introduced to the same transform to allow configuring the query priority (Python) ([BEAM-12913](https://issues.apache.org/jira/browse/BEAM-12913)).
+* `ReadFromBigQuery` and `ReadAllFromBigQuery` now run queries with BATCH priority by default. The `query_priority` parameter is introduced to the same transforms to allow configuring the query priority (Python) ([BEAM-12913](https://issues.apache.org/jira/browse/BEAM-12913)).
 
 ## New Features / Improvements
 
@@ -123,7 +123,7 @@
   * Code depending on beam imports need to include v2 on the module path.
     * Fix by'v2' to the import paths, turning  `.../sdks/go/...` to `.../sdks/v2/go/...`
   * No other code change should be required to use v2.33.0 of the Go SDK.
-  
+
 ## Deprecations
 
 * X behavior is deprecated and will be removed in X versions ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)).

build.gradle.kts

@@ -42,6 +42,7 @@ tasks.rat {
   val exclusions = mutableListOf(
     // Ignore files we track but do not distribute
     "**/.github/**/*",
+    "**/.gitkeep",
     "gradlew",
     "gradlew.bat",
     "gradle/wrapper/gradle-wrapper.properties",
@@ -117,7 +118,14 @@ tasks.rat {
     "sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/yarn.lock",
 
     // Sample text file for Java quickstart
-    "sdks/java/maven-archetypes/examples/sample.txt"
+    "sdks/java/maven-archetypes/examples/sample.txt",
+
+    // Ignore Flutter autogenerated files for Playground
+    "playground/frontend/.metadata",
+    "playground/frontend/pubspec.lock",
+
+    // Ignore .gitkeep file
+    "**/.gitkeep"
   )
 
   // Add .gitignore excludes to the Apache Rat exclusion list. We re-create the behavior

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -2337,7 +2337,7 @@ class BeamModulePlugin implements Plugin<Project> {
           def distTarBall = "${pythonRootDir}/build/apache-beam.tar.gz"
           project.exec {
             executable 'sh'
-            args '-c', ". ${project.ext.envdir}/bin/activate && pip install --retries 10 ${distTarBall}[gcp,test,aws,azure]"
+            args '-c', ". ${project.ext.envdir}/bin/activate && pip install --retries 10 ${distTarBall}[gcp,test,aws,azure,dataframe]"
           }
         }
       }

examples/notebooks/tour-of-beam/dataframes.ipynb

@@ -65,7 +65,7 @@
         "[Beam DataFrames overview](https://beam.apache.org/documentation/dsls/dataframes/overview) page.\n",
         "\n",
         "First, we need to install Apache Beam with the `interactive` extra for the Interactive runner.",
-        "We also need `pandas` for this notebook, but the Interactive runner already depends on it."
+        "We also need to install a version of `pandas` supported by the DataFrame API, which we can get with the `dataframe` extra in Beam 2.34.0 and newer."
       ],
       "metadata": {
         "id": "hDuXLLSZnI1D"
@@ -75,7 +75,7 @@
       "cell_type": "code",
       "execution_count": null,
       "source": [
-        "%pip install --quiet apache-beam[interactive]"
+        "%pip install --quiet apache-beam[interactive,dataframe]"
       ],
       "outputs": [],
       "metadata": {
@@ -663,7 +663,7 @@
         "\n",
         "> ℹ️ It's recommended to **only** do this if you need to use a Pandas operation that is\n",
         "> [not supported in Beam DataFrames](https://beam.apache.org/documentation/dsls/dataframes/differences-from-pandas/#classes-of-unsupported-operations).\n",
-        "> Converting a PCollection into a Pandas DataFrame consolidates elements from potentially multiple workers into a single worker, which could create a performance bottleneck.\n"
+        "> Converting a PCollection into a Pandas DataFrame consolidates elements from potentially multiple workers into a single worker, which could create a performance bottleneck.\n",
         "\n",
         "> ⚠️ Pandas DataFrames are in-memory data structures, so make sure all the elements in the PCollection fit into memory.\n",
         "> If they don't fit into memory, consider yielding multiple DataFrame elements via\n",

model/pipeline/src/main/proto/beam_runner_api.proto

@@ -1718,7 +1718,14 @@ message LabelledPayload {
     string string_value = 2;
     bool bool_value = 3;
     double double_value = 4;
+    int64 int_value = 5;
   }
+
+  // (Required) The key identifies the actual content of the metadata.
+  string key = 6;
+
+  // (Required) The namespace describes the context that specified the key.
+  string namespace = 7;
 }
 
 // Static display data associated with a pipeline component. Display data is

model/pipeline/src/main/proto/metrics.proto

@@ -420,6 +420,10 @@ message MonitoringInfo {
     BIGTABLE_PROJECT_ID = 20 [(label_props) = { name: "BIGTABLE_PROJECT_ID"}];
     INSTANCE_ID = 21 [(label_props) = { name: "INSTANCE_ID"}];
     TABLE_ID = 22 [(label_props) = { name: "TABLE_ID"}];
+    SPANNER_PROJECT_ID = 23 [(label_props) = { name: "SPANNER_PROJECT_ID"}];
+    SPANNER_DATABASE_ID = 24 [(label_props) = { name: "SPANNER_DATABASE_ID"}];
+    SPANNER_INSTANCE_ID = 25 [(label_props) = { name: "SPANNER_INSTANCE_ID" }];
+    SPANNER_QUERY_NAME = 26 [(label_props) = { name: "SPANNER_QUERY_NAME" }];
   }
 
   // A set of key and value labels which define the scope of the metric. For

playground/backend/cmd/server/server.go

@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"beam.apache.org/playground/backend/pkg/executors"
+)
+
+func main() {
+	_ = executors.GoExecutor{}
+}

playground/backend/containers/go/Dockerfile

@@ -0,0 +1,19 @@
+###############################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+#Dokerfile to set up the Beam Go SDK

playground/backend/containers/java/Dockerfile

@@ -0,0 +1,19 @@
+###############################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+#Dokerfile to set up the Beam Java SDK

playground/backend/go.mod

@@ -0,0 +1,18 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+module beam.apache.org/playground/backend
+
+go 1.16

playground/backend/pkg/executors/executor.go

@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Interface for all executors (Java/Python/Go/SCIO)
+package executors
+
+type executor interface {
+	// Validate validates executable file.
+	// Return result of validation (true/false) and error if it occurs
+	Validate(filePath string) (bool, error)
+
+	// Compile compiles executable file.
+	// Return error if it occurs
+	Compile(filePath string) error
+
+	// Run runs executable file.
+	// Return logs and error if it occurs
+	Run(filePath string) (string, error)
+}

playground/backend/pkg/executors/goexecutor.go

@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Executor for Go
+package executors
+
+type GoExecutor struct{}
+
+func (goExec GoExecutor) Validate(filePath string) (bool, error) {
+	return true, nil
+}
+
+func (goExec GoExecutor) Compile(filePath string) error {
+	return nil
+}
+
+func (goExec GoExecutor) Run(filePath string) (string, error) {
+	return "", nil
+}

playground/backend/pkg/executors/javaexecutor.go

@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Executor for Java
+package executors
+
+type JavaExecutor struct{}
+
+func (javaExec JavaExecutor) Validate(filePath string) (bool, error) {
+	return true, nil
+}
+
+func (javaExec JavaExecutor) Compile(filePath string) error {
+	return nil
+}
+
+func (javaExec JavaExecutor) Run(filePath string) (string, error) {
+	return "", nil
+}

playground/frontend/.metadata

@@ -0,0 +1,10 @@
+# This file tracks properties of this Flutter project.
+# Used by Flutter tool to assess capabilities and perform upgrades etc.
+#
+# This file should be version controlled and should not be manually edited.
+
+version:
+  revision: ffb2ecea5223acdd139a5039be2f9c796962833d
+  channel: stable
+
+project_type: app

playground/frontend/README.md

@@ -0,0 +1,47 @@
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+-->
+
+# Apache Beam Playground
+
+## About
+
+Apache Beam is an open-source, unified model for defining parallel processing pipelines for batch and streaming data.
+It provides a portable API layer for building sophisticated data-parallel processing pipelines that may be executed across a diversity of execution engines, or runners.
+
+## Getting Started
+
+Website development requires [Flutter](https://flutter.dev/docs/get-started/install) installed.
+
+The following command is used to build and serve the website locally:
+
+`$ flutter run`
+
+Run the following command to generate a release build:
+
+`flutter build web`
+
+Playground tests may be run using this command:
+
+`flutter test`
+
+Dart code should follow next [code style](https://dart-lang.github.io/linter/lints/index.html). Code may be analyzed using this command:
+
+`flutter analyze`
+
+The full list of command can be found [here](https://flutter.dev/docs/reference/flutter-cli)