Update project settings and dependencies

This adds a Docker container to run locally for development and for
testing, as well as updates various dependencies to latest versions,
and moves to CircleCI for testing and publishing.

This commit differs from the prior commits by using sbt 1.0.4 which
fixes a problem with `sbt publish`.
See: sbt/librarymanagement#183

.gitignore

@@ -2,3 +2,5 @@ target/
 .idea/
 .idea_modules/
 .DS_Store
+.sbt/
+.ivy2/

.travis.yml

@@ -3,9 +3,10 @@ sudo: false
 cache:
   directories:
     - $HOME/.ivy2
+    - $HOME/.sbt
 matrix:
   include:
-    - jdk: openjdk7
+    - jdk: openjdk8
       scala: 2.11.8
       python: 3.5
       env: TEST_SPARK_VERSION="2.1.0" AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=foo

Dockerfile

@@ -1,7 +1,7 @@
 FROM openjdk:8
 
 ENV SCALA_VERSION=2.12.4 \
-    SBT_VERSION=1.0.3
+    SBT_VERSION=1.0.4
 
 # Install Scala.
 RUN \

README.md

@@ -3,24 +3,39 @@
 [![CircleCi](https://circleci.com/gh/mozilla/moztelemetry.svg?style=shield&circle-token=3fff2168f7d8da61b47bd1481c4fcb984ec88ef5)](https://circleci.com/gh/mozilla/moztelemetry)
 
 # moztelemetry
+
 Mozilla's Telemetry API for Scala
 
 ## Using moztelemetry
+
 In SBT:
 ```
 resolvers += "S3 local maven snapshots" at "s3://net-mozaws-data-us-west-2-ops-mavenrepo/snapshots"
 libraryDependencies += "com.mozilla.telemetry" %% "moztelemetry" % "1.0-SNAPSHOT"
 ```
 
 ## Testing
-To run the tests you have to start a mock S3 service first with moto:
+
+To run the tests, build the docker container and run the tests.
+
+Build the container. You only have to do this once or when you update the Dockerfile:
+
+```
+docker build -t moztelemetry .
+```
+
+Run the tests in the docker container:
 
 ```
-pip install moto
-moto_server s3 -p 8001 &
-AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=foo sbt test
+./bin/test
+```
+
+Other test tasks can by run by passing the task through the test script, e.g.:
+
+```
+./bin/test "testOnly com.mozilla.telemetry.stats.StatsTest"
 ```
 
 ## Publishing snapshots
-Snapshots will now be published to our local maven repo in s3 on every commit merged into master via a circleci build
 
+Snapshots will now be published to our local maven repo in s3 on every commit merged into master via a circleci build

build.sbt

@@ -6,32 +6,26 @@ organization := "com.mozilla.telemetry"
 
 scalaVersion := "2.11.8"
 
-sparkVersion := "2.1.0"
-
-sparkComponents ++= Seq("core")
+val sparkVersion = "2.2.0"
 
 resolvers += Resolver.bintrayRepo("findify", "maven")
 
 libraryDependencies ++= Seq(
+  "org.apache.spark" %% "spark-core" % sparkVersion % "provided",
   "org.scalatest" %% "scalatest" % "2.2.6" % "test",
-  "org.apache.commons" % "commons-io" % "1.3.2" % "test",
+  "commons-io" % "commons-io" % "1.3.2" % "test",
   "com.github.seratch" %% "awscala" % "0.5.+",
   "com.amazonaws" % "aws-java-sdk" % "1.11.83",
   "com.google.protobuf" % "protobuf-java" % "2.5.0"
 )
+
 /*
  The HBase client requires protobuf-java 2.5.0 but scalapb uses protobuf-java 3.x
  so we have to force the dependency here. This should be fine as we are using only
  version 2 of the protobuf spec.
 */
 dependencyOverrides += "com.google.protobuf" % "protobuf-java" % "2.5.0"
 
-// Shade PB files
-assemblyShadeRules in assembly := Seq(
-  ShadeRule.rename("com.google.protobuf.**" -> "shadeproto.@1").inAll,
-  ShadeRule.rename("com.trueaccord.scalapb.**" -> "shadescalapb.@1").inAll
-) 
-
 // Compile proto files
 PB.targets in Compile := Seq(
   scalapb.gen() -> (sourceManaged in Compile).value
@@ -40,8 +34,6 @@ PB.targets in Compile := Seq(
 // Exclude generated classes from the coverage
 coverageExcludedPackages := "com\\.mozilla\\.telemetry\\.heka\\.(Field|Message|Header)"
 
-credentials += Credentials(Path.userHome / ".ivy2" / ".sbtcredentials")
-
 publishMavenStyle := true
 
 publishTo := {

circle.yml

@@ -1,31 +1,23 @@
 machine:
   pre:
-    # Install sbt 0.13.16
+    # Install sbt 1.0.4
     - sudo apt-get install openjdk-8-jdk
-    - wget -q https://dl.bintray.com/sbt/debian/sbt-0.13.16.deb
-    - sudo dpkg -i sbt-0.13.16.deb
+    - wget -q https://dl.bintray.com/sbt/debian/sbt-1.0.4.deb
+    - sudo dpkg -i sbt-1.0.4.deb
   cache_directories:
     - "~/.ivy2"
     - "~/.sbt"
+  services:
+    - docker
 
 dependencies:
-  pre:
-    - sudo apt-get update
-    - sudo apt-get install libffi-dev libssl-dev python-dev
-    - sudo apt-get install python
-    - python --version
-    - curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-    - sudo python get-pip.py
-    - sudo pip install --upgrade pip
-    - sudo pip install --upgrade setuptools
-    - sudo pip install pyopenssl ndg-httpsclient pyasn1
-    - sudo pip install moto --ignore-installed six
-    - sudo pip install flask
-    - moto_server s3 -p 8001 &
+  override:
+    - docker info
+    - docker build --rm=false -t circleci/moztelemetry:$CIRCLE_SHA1 .
 
 test:
   override:
-    - 'true'
+    - docker run -t -i -p 8001:8001 circleci/moztelemetry:$CIRCLE_SHA1 ./bin/run_tests
 
 deployment:
   latest:

project/build.properties

@@ -0,0 +1 @@
+sbt.version=1.0.4

project/plugins.sbt

@@ -1,13 +1,11 @@
 resolvers += "bintray-spark-packages" at "https://dl.bintray.com/spark-packages/maven/"
 
-addSbtPlugin("org.spark-packages" % "sbt-spark-package" % "0.2.5")
+addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.1")
 
-addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.0")
+addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0")
 
-addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.8.0")
-
-addSbtPlugin("com.thesamet" % "sbt-protoc" % "0.99.3")
+addSbtPlugin("com.thesamet" % "sbt-protoc" % "0.99.12")
 
 addSbtPlugin("com.frugalmechanic" % "fm-sbt-s3-resolver" % "0.12.0")
 
-libraryDependencies += "com.trueaccord.scalapb" %% "compilerplugin" % "0.5.47"
+libraryDependencies += "com.trueaccord.scalapb" %% "compilerplugin" % "0.6.6"

src/main/scala/com/mozilla/telemetry/heka/package.scala

@@ -19,7 +19,7 @@ package object heka {
   private def field(f: Field): Any = {
     // I am assuming there is only one value
     f.getValueType match {
-      case Field.ValueType.BYTES => {
+      case Field.ValueTypeEnum.BYTES => {
         val bytes = f.valueBytes(0)
         // Our JSON bytes fields sometimes contain non-UTF8 strings that can
         // still be parsed as JSON. For now, we attempt to coerce all bytes
@@ -28,10 +28,10 @@ package object heka {
         // for details.
         bytes.toStringUtf8
       }
-      case Field.ValueType.STRING => f.valueString(0)
-      case Field.ValueType.BOOL => f.valueBool(0)
-      case Field.ValueType.DOUBLE => f.valueDouble(0)
-      case Field.ValueType.INTEGER => f.valueInteger(0)
+      case Field.ValueTypeEnum.STRING => f.valueString(0)
+      case Field.ValueTypeEnum.BOOL => f.valueBool(0)
+      case Field.ValueTypeEnum.DOUBLE => f.valueDouble(0)
+      case Field.ValueTypeEnum.INTEGER => f.valueInteger(0)
       case _ => assert(false)
     }
   }
@@ -42,19 +42,19 @@ package object heka {
     def apply (uuid: String, fieldsMap: Map[String, Any], payload: Option[String], timestamp: Long=0): Message = {
       val fields = fieldsMap.toList.map{
         case (k: String, v: ByteString) => {
-          Field(k, Some(Field.ValueType.BYTES), valueBytes=Seq(v))
+          Field(k, Some(Field.ValueTypeEnum.BYTES), valueBytes=Seq(v))
         }
         case (k: String, v: String) => {
-          Field(k, Some(Field.ValueType.STRING), valueString=Seq(v))
+          Field(k, Some(Field.ValueTypeEnum.STRING), valueString=Seq(v))
         }
         case (k: String, v: Boolean) => {
-          Field(k, Some(Field.ValueType.BOOL), valueBool=Seq(v))
+          Field(k, Some(Field.ValueTypeEnum.BOOL), valueBool=Seq(v))
         }
         case (k: String, v: Double) => {
-          Field(k, Some(Field.ValueType.DOUBLE), valueDouble=Seq(v))
+          Field(k, Some(Field.ValueTypeEnum.DOUBLE), valueDouble=Seq(v))
         }
         case (k: String, v: Long) => {
-          Field(k, Some(Field.ValueType.INTEGER), valueInteger=Seq(v))
+          Field(k, Some(Field.ValueTypeEnum.INTEGER), valueInteger=Seq(v))
         }
       }.toSeq
       Message(ByteString.copyFromUtf8(uuid), timestamp, payload=payload, fields=fields)