Twitter redesign to move the component to it's separate project. (#14902

) * Twitter redesign to move the component to it's separate project. * Removing unneeded and duplicate tests * GoPortable PreCommit and Java PreCommit correction. * GoPortable PreCommit and Java PreCommit correction. * Fix execute task * Fix execute task * Fix execute task * re ordering the code. * Spark Runner Pre commit failing * Spark Runner removal
apache · Jun 2, 2021 · d0c3dd7 · d0c3dd7
1 parent ed8554d
commit d0c3dd7
Show file tree

Hide file tree

Showing 13 changed files with 117 additions and 9 deletions.
diff --git a/build.gradle.kts b/build.gradle.kts
@@ -145,6 +145,7 @@ task("javaPreCommit") {
   dependsOn(":sdks:java:core:buildNeeded")
   dependsOn(":sdks:java:core:buildDependents")
   dependsOn(":examples:java:preCommit")
+  dependsOn(":examples:java:twitter:preCommit")
   dependsOn(":sdks:java:extensions:sql:jdbc:preCommit")
   dependsOn(":sdks:java:javadoc:allJavadoc")
   dependsOn(":runners:direct-java:needsRunnerTests")

diff --git a/examples/java/build.gradle b/examples/java/build.gradle
@@ -91,8 +91,6 @@ dependencies {
   compile "org.apache.commons:commons-lang3:3.9"
   compile "org.apache.httpcomponents:httpclient:4.5.13"
   compile "org.apache.httpcomponents:httpcore:4.4.13"
-  compile ("org.twitter4j:twitter4j-stream:4.0.7")
-  compile ("org.twitter4j:twitter4j-core:4.0.7")
   testCompile project(path: ":runners:direct-java", configuration: "shadow")
   testCompile project(":sdks:java:io:google-cloud-platform")
   testCompile project(":sdks:java:extensions:ml")

diff --git a/examples/java/twitter/build.gradle b/examples/java/twitter/build.gradle
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import groovy.json.JsonOutput
+
+plugins {
+    id 'java'
+    id 'org.apache.beam.module'
+    id 'com.github.johnrengelman.shadow'
+}
+
+applyJavaNature(
+        exportJavadoc: false,
+        automaticModuleName: 'org.apache.beam.examples.twitterstreamgenerator',
+)
+provideIntegrationTestingDependencies()
+enableJavaPerformanceTesting()
+
+description = "Apache Beam :: Examples :: Java :: Twitter "
+ext.summary = """Apache Beam SDK provides a simple, Java-based
+interface for processing virtually any size data. This
+artifact includes all Apache Beam Java SDK examples."""
+
+/** Define the list of runners which execute a precommit test.
+ * Some runners are run from separate projects, see the preCommit task below
+ * for details.
+ */
+def preCommitRunners = ["directRunner", "flinkRunner"]
+for (String runner : preCommitRunners) {
+    configurations.create(runner + "PreCommit")
+}
+
+dependencies {
+    compile enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
+    compile project(path: ":sdks:java:core", configuration: "shadow")
+    compile library.java.joda_time
+    compile library.java.slf4j_api
+    provided library.java.commons_io
+    provided library.java.commons_csv
+    runtime project(path: ":runners:direct-java", configuration: "shadow")
+    compile ("org.twitter4j:twitter4j-stream:4.0.7")
+    compile ("org.twitter4j:twitter4j-core:4.0.7")
+    testCompile project(path: ":runners:direct-java", configuration: "shadow")
+    testCompile library.java.hamcrest_core
+    testCompile library.java.hamcrest_library
+    testCompile library.java.junit
+    testCompile library.java.mockito_core
+    testCompile library.java.testcontainers_gcloud
+
+    // Add dependencies for the PreCommit configurations
+    // For each runner a project level dependency on the examples project.
+    for (String runner : preCommitRunners) {
+        delegate.add(runner + "PreCommit", project(":examples:java:twitter"))
+        delegate.add(runner + "PreCommit", project(path: ":examples:java:twitter", configuration: "testRuntime"))
+    }
+    directRunnerPreCommit project(path: ":runners:direct-java", configuration: "shadow")
+    flinkRunnerPreCommit project(":runners:flink:${project.ext.latestFlinkVersion}")
+}
+
+/*
+ * Create a ${runner}PreCommit task for each runner which runs a set
+ * of integration tests for WordCount and WindowedWordCount.
+ */
+def preCommitRunnerClass = [
+        directRunner: "org.apache.beam.runners.direct.DirectRunner",
+        flinkRunner: "org.apache.beam.runners.flink.TestFlinkRunner",
+]
+
+for (String runner : preCommitRunners) {
+    tasks.create(name: runner + "PreCommit", type: Test) {
+        def preCommitBeamTestPipelineOptions = [
+                "--runner=" + preCommitRunnerClass[runner],
+        ]
+        classpath = configurations."${runner}PreCommit"
+        forkEvery 1
+        maxParallelForks 4
+        systemProperty "beamTestPipelineOptions", JsonOutput.toJson(preCommitBeamTestPipelineOptions)
+    }
+}
+
+/* Define a common precommit task which depends on all the individual precommits. */
+task preCommit() {
+    for (String runner : preCommitRunners) {
+        dependsOn runner + "PreCommit"
+    }
+}
+
+task execute (type:JavaExec) {
+    main = "org.apache.beam.examples.twitterstreamgenerator.TwitterStream"
+}
diff --git a/...complete/twitterstreamgenerator/README.md → ...examples/twitterstreamgenerator/README.md b/...complete/twitterstreamgenerator/README.md → ...examples/twitterstreamgenerator/README.md
@@ -34,6 +34,7 @@ This directory contains an example pipelines for how to perform continues stream
 
 - Java 8
 - Twitter developer app account and streaming credentials.
+- Direct runner or Flink runner.
 
 This section describes what is needed to get the example up and running.
 

diff --git a/...rstreamgenerator/ReadFromTwitterDoFn.java → ...rstreamgenerator/ReadFromTwitterDoFn.java b/...rstreamgenerator/ReadFromTwitterDoFn.java → ...rstreamgenerator/ReadFromTwitterDoFn.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.examples.complete.twitterstreamgenerator;
+package org.apache.beam.examples.twitterstreamgenerator;
 
 import java.io.IOException;
 import java.io.Serializable;

diff --git a/...twitterstreamgenerator/TwitterConfig.java → ...twitterstreamgenerator/TwitterConfig.java b/...twitterstreamgenerator/TwitterConfig.java → ...twitterstreamgenerator/TwitterConfig.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.examples.complete.twitterstreamgenerator;
+package org.apache.beam.examples.twitterstreamgenerator;
 
 import java.io.Serializable;
 import java.util.ArrayList;

diff --git a/...terstreamgenerator/TwitterConnection.java → ...terstreamgenerator/TwitterConnection.java b/...terstreamgenerator/TwitterConnection.java → ...terstreamgenerator/TwitterConnection.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.examples.complete.twitterstreamgenerator;
+package org.apache.beam.examples.twitterstreamgenerator;
 
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;

diff --git a/...ete/twitterstreamgenerator/TwitterIO.java → ...les/twitterstreamgenerator/TwitterIO.java b/...ete/twitterstreamgenerator/TwitterIO.java → ...les/twitterstreamgenerator/TwitterIO.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.examples.complete.twitterstreamgenerator;
+package org.apache.beam.examples.twitterstreamgenerator;
 
 import java.util.ArrayList;
 import java.util.List;

diff --git a/...twitterstreamgenerator/TwitterStream.java → ...twitterstreamgenerator/TwitterStream.java b/...twitterstreamgenerator/TwitterStream.java → ...twitterstreamgenerator/TwitterStream.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.examples.complete.twitterstreamgenerator;
+package org.apache.beam.examples.twitterstreamgenerator;
 
 import java.util.Arrays;
 import org.apache.beam.sdk.Pipeline;

diff --git a/.../twitterstreamgenerator/package-info.java → .../twitterstreamgenerator/package-info.java b/.../twitterstreamgenerator/package-info.java → .../twitterstreamgenerator/package-info.java
@@ -17,4 +17,4 @@
  */
 
 /** Kafka to Pubsub template. */
-package org.apache.beam.examples.complete.twitterstreamgenerator;
+package org.apache.beam.examples.twitterstreamgenerator;
diff --git a/...eamgenerator/ReadFromTwitterDoFnTest.java → ...eamgenerator/ReadFromTwitterDoFnTest.java b/...eamgenerator/ReadFromTwitterDoFnTest.java → ...eamgenerator/ReadFromTwitterDoFnTest.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.examples.complete.twitterstreamgenerator;
+package org.apache.beam.examples.twitterstreamgenerator;
 
 import static org.junit.Assert.assertArrayEquals;
 import static org.mockito.Mockito.when;

diff --git a/sdks/java/io/hadoop-format/build.gradle b/sdks/java/io/hadoop-format/build.gradle
@@ -68,6 +68,7 @@ dependencies {
   testCompile project(path: ":sdks:java:testing:test-utils", configuration: "testRuntime")
   testCompile project(":sdks:java:io:jdbc")
   testCompile project(path: ":examples:java", configuration: "testRuntime")
+  testCompile project(path: ":examples:java:twitter", configuration: "testRuntime")
   testCompile "org.elasticsearch.plugin:transport-netty4-client:$elastic_search_version"
   testCompile library.java.testcontainers_elasticsearch
   testCompile "org.elasticsearch.client:elasticsearch-rest-high-level-client:$elastic_search_version"
@@ -103,6 +104,7 @@ dependencies {
   delegate.add("sparkRunner", project(path: ":sdks:java:io:hadoop-format", configuration: "testRuntime"))
 
   sparkRunner project(path: ":examples:java", configuration: "testRuntime")
+  sparkRunner project(path: ":examples:java:twitter", configuration: "testRuntime")
   sparkRunner project(":runners:spark:2")
   sparkRunner project(":sdks:java:io:hadoop-file-system")
   sparkRunner library.java.spark_streaming

diff --git a/settings.gradle.kts b/settings.gradle.kts
@@ -51,6 +51,7 @@ include(":release:go-licenses:java")
 include(":release:go-licenses:py")
 
 include(":examples:java")
+include(":examples:java:twitter")
 include(":examples:kotlin")
 include(":model:fn-execution")
 include(":model:job-management")
-Original file line number
+Diff line change
@@ Expand Up @@
     - Java 8
     - Twitter developer app account and streaming credentials.
+    - Direct runner or Flink runner.
     This section describes what is needed to get the example up and running.
@@ Expand Down @@