Merge pull request apache#482 from tdas/streaming-example-fix

Added StreamingContext.awaitTermination to streaming examples

StreamingContext.start() currently starts a non-daemon thread which prevents termination of a Spark Streaming program even if main function has exited. Since the expected behavior of a streaming program is to run until explicitly killed, this was sort of fine when spark streaming applications are launched from the command line. However, when launched in Yarn-standalone mode, this did not work as the driver effectively got terminated when the main function exits. So SparkStreaming examples did not work on Yarn.

This addition to the examples ensures that the examples work on Yarn and also ensures that everyone learns that StreamingContext.awaitTermination() being necessary for SparkStreaming programs to wait.

The true bug-fix of making sure all threads by Spark Streaming are daemon threads is left for post-0.9.
(cherry picked from commit 0367981)

Signed-off-by: Patrick Wendell <[email protected]>

examples/src/main/java/org/apache/spark/streaming/examples/JavaFlumeEventCount.java

@@ -70,5 +70,6 @@ public String call(Long in) {
     }).print();
 
     ssc.start();
+    ssc.awaitTermination();
   }
 }

examples/src/main/java/org/apache/spark/streaming/examples/JavaKafkaWordCount.java

@@ -104,5 +104,6 @@ public Integer call(Integer i1, Integer i2) {
 
     wordCounts.print();
     jssc.start();
+    jssc.awaitTermination();
   }
 }

examples/src/main/java/org/apache/spark/streaming/examples/JavaNetworkWordCount.java

@@ -84,5 +84,6 @@ public Integer call(Integer i1, Integer i2) {
 
     wordCounts.print();
     ssc.start();
+    ssc.awaitTermination();
   }
 }

examples/src/main/java/org/apache/spark/streaming/examples/JavaQueueStream.java

@@ -80,5 +80,6 @@ public Integer call(Integer i1, Integer i2) {
 
     reducedStream.print();
     ssc.start();
+    ssc.awaitTermination();
   }
 }

examples/src/main/scala/org/apache/spark/streaming/examples/ActorWordCount.scala

@@ -171,5 +171,6 @@ object ActorWordCount {
     lines.flatMap(_.split("\\s+")).map(x => (x, 1)).reduceByKey(_ + _).print()
 
     ssc.start()
+    ssc.awaitTermination()
   }
 }

examples/src/main/scala/org/apache/spark/streaming/examples/FlumeEventCount.scala

@@ -60,5 +60,6 @@ object FlumeEventCount {
     stream.count().map(cnt => "Received " + cnt + " flume events." ).print()
 
     ssc.start()
+    ssc.awaitTermination()
   }
 }

examples/src/main/scala/org/apache/spark/streaming/examples/HdfsWordCount.scala

@@ -50,6 +50,7 @@ object HdfsWordCount {
     val wordCounts = words.map(x => (x, 1)).reduceByKey(_ + _)
     wordCounts.print()
     ssc.start()
+    ssc.awaitTermination()
   }
 }
 

examples/src/main/scala/org/apache/spark/streaming/examples/KafkaWordCount.scala

@@ -61,6 +61,7 @@ object KafkaWordCount {
     wordCounts.print()
 
     ssc.start()
+    ssc.awaitTermination()
   }
 }
 

examples/src/main/scala/org/apache/spark/streaming/examples/MQTTWordCount.scala

@@ -101,5 +101,6 @@ object MQTTWordCount {
     val wordCounts = words.map(x => (x, 1)).reduceByKey(_ + _)
     wordCounts.print()
     ssc.start()
+    ssc.awaitTermination()
   }
 }

examples/src/main/scala/org/apache/spark/streaming/examples/NetworkWordCount.scala

@@ -54,5 +54,6 @@ object NetworkWordCount {
     val wordCounts = words.map(x => (x, 1)).reduceByKey(_ + _)
     wordCounts.print()
     ssc.start()
+    ssc.awaitTermination()
   }
 }

examples/src/main/scala/org/apache/spark/streaming/examples/RawNetworkGrep.scala

@@ -61,5 +61,6 @@ object RawNetworkGrep {
     union.filter(_.contains("the")).count().foreachRDD(r =>
       println("Grep count: " + r.collect().mkString))
     ssc.start()
+    ssc.awaitTermination()
   }
 }

examples/src/main/scala/org/apache/spark/streaming/examples/RecoverableNetworkWordCount.scala

@@ -114,5 +114,6 @@ object RecoverableNetworkWordCount {
         createContext(master, ip, port, outputPath)
       })
     ssc.start()
+    ssc.awaitTermination()
   }
 }

examples/src/main/scala/org/apache/spark/streaming/examples/StatefulNetworkWordCount.scala

@@ -65,5 +65,6 @@ object StatefulNetworkWordCount {
     val stateDstream = wordDstream.updateStateByKey[Int](updateFunc)
     stateDstream.print()
     ssc.start()
+    ssc.awaitTermination()
   }
 }

examples/src/main/scala/org/apache/spark/streaming/examples/TwitterAlgebirdCMS.scala

@@ -110,5 +110,6 @@ object TwitterAlgebirdCMS {
     })
 
     ssc.start()
+    ssc.awaitTermination()
   }
 }

examples/src/main/scala/org/apache/spark/streaming/examples/TwitterAlgebirdHLL.scala

@@ -87,5 +87,6 @@ object TwitterAlgebirdHLL {
     })
 
     ssc.start()
+    ssc.awaitTermination()
   }
 }

examples/src/main/scala/org/apache/spark/streaming/examples/TwitterPopularTags.scala

@@ -69,5 +69,6 @@ object TwitterPopularTags {
     })
 
     ssc.start()
+    ssc.awaitTermination()
   }
 }

examples/src/main/scala/org/apache/spark/streaming/examples/ZeroMQWordCount.scala

@@ -91,5 +91,6 @@ object ZeroMQWordCount {
     val wordCounts = words.map(x => (x, 1)).reduceByKey(_ + _)
     wordCounts.print()
     ssc.start()
+    ssc.awaitTermination()
   }
 }