Merge from twitter/storehaus

.travis.yml

@@ -1,11 +1,12 @@
 language: scala
 scala:
   - 2.10.0
-  - 2.9.2
+  - 2.9.3
 before_script:
   - mysql -u root -e "create database storehaus_test;"
   - mysql -u root -e "create user 'storehaususer'@'localhost' identified by 'test1234';"
   - mysql -u root -e "grant all on storehaus_test.* to 'storehaususer'@'localhost';"
 services:
   - redis-server
-  - memcache
+  - memcache
+script: umask 0022 && sbt ++$TRAVIS_SCALA_VERSION test

CHANGES.md

@@ -1,5 +1,22 @@
 # storehaus #
 
+### Version.0.5.1 ###
+
+* Add storehaus-hbase and upgrade to bijection 0.5.3: https://github.com/twitter/storehaus/pull/139
+* Fix mutable TTL cache bug: https://github.com/twitter/storehaus/pull/136
+
+### Version.0.5.0 ###
+
+* Reuse prepared statements in mysql: https://github.com/twitter/storehaus/issues/93
+* storehaus-testing module: https://github.com/twitter/storehaus/pull/115
+* cache ttl is now a duration, vs a time: https://github.com/twitter/storehaus/pull/100
+* improve performance of CollectionOps: https://github.com/twitter/storehaus/pull/117
+* Augment memcachestore with common functions: https://github.com/twitter/storehaus/pull/121
+* bump twitter-util and finagle versions: https://github.com/twitter/storehaus/pull/125
+* Upgrade to scala 2.9.3, algebird 0.2.0 and Bijection 0.5.2: https://github.com/twitter/storehaus/pull/126
+
+Thanks to Doug Tangren, Ruban Monu, Ximing Yu, Ryan LeCompte, Sam Ritchie and Oscar Boykin for contributions!
+
 ### Version.0.4.0 ###
 
 * Storehaus-Mysql support for numeric types

README.md

@@ -79,6 +79,7 @@ Storehaus provides a number of modules wrapping existing key-value stores. Enric
   * [Storehaus-memcache](http://twitter.github.com/storehaus/#com.twitter.storehaus.memcache.MemcacheStore) (wraps Twitter's [finagle-memcached](https://github.com/twitter/finagle/tree/master/finagle-memcached) library)
   * [Storehaus-mysql](http://twitter.github.com/storehaus/#com.twitter.storehaus.mysql.MySQLStore) (wraps Twitter's [finagle-mysql](https://github.com/twitter/finagle/tree/master/finagle-mysql) library)
   * [Storehaus-redis](http://twitter.github.com/storehaus/#com.twitter.storehaus.redis.RedisStore) (wraps Twitter's [finagle-redis](https://github.com/twitter/finagle/tree/master/finagle-redis) library)
+  * [Storehaus-hbase](http://twitter.github.com/storehaus/#com.twitter.storehaus.hbase.HBaseStore)
 
 #### Planned Modules
 
@@ -94,22 +95,26 @@ See the [current API documentation](http://twitter.github.com/storehaus) for mor
 
 ## Maven
 
-Storehaus modules are available on maven central. The current groupid and version for all modules is, respectively, `"com.twitter"` and  `0.4.0`.
+Storehaus modules are available on maven central. The current groupid and version for all modules is, respectively, `"com.twitter"` and  `0.5.1`.
 
 Current published artifacts are
 
-* `storehaus-core_2.9.2`
+* `storehaus-core_2.9.3`
 * `storehaus-core_2.10`
-* `storehaus-algebra_2.9.2`
+* `storehaus-algebra_2.9.3`
 * `storehaus-algebra_2.10`
-* `storehaus-memcache_2.9.2`
+* `storehaus-memcache_2.9.3`
 * `storehaus-memcache_2.10`
-* `storehaus-mysql_2.9.2`
+* `storehaus-mysql_2.9.3`
 * `storehaus-mysql_2.10`
-* `storehaus-redis_2.9.2`
+* `storehaus-hbase_2.9.3`
+* `storehaus-hbase_2.10`
+* `storehaus-redis_2.9.3`
 * `storehaus-redis_2.10`
-* `storehaus-cache_2.9.2`
+* `storehaus-cache_2.9.3`
 * `storehaus-cache_2.10`
+* `storehaus-testing_2.9.3`
+* `storehaus-testing_2.10`
 
 The suffix denotes the scala version.
 

project/Build.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2013 Twitter inc.
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
 package storehaus
 
 import sbt._
@@ -8,6 +24,13 @@ import com.typesafe.tools.mima.plugin.MimaPlugin.mimaDefaultSettings
 import com.typesafe.tools.mima.plugin.MimaKeys.previousArtifact
 
 object StorehausBuild extends Build {
+  def withCross(dep: ModuleID) =
+    dep cross CrossVersion.binaryMapped {
+      case "2.9.3" => "2.9.2" // TODO: hack because twitter hasn't built things against 2.9.3
+      case version if version startsWith "2.10" => "2.10" // TODO: hack because sbt is broken
+      case x => x
+    }
+
   val extraSettings =
     Project.defaultSettings ++ releaseSettings ++ Boilerplate.settings ++ mimaDefaultSettings
 
@@ -19,7 +42,7 @@ object StorehausBuild extends Build {
       logLevel in Test := Level.Info
     ) else Seq.empty[Project.Setting[_]]
 
-  val testCleanup = extraSettings ++ Seq(
+  val testCleanup = Seq(
     testOptions in Test += Tests.Cleanup { loader =>
       val c = loader.loadClass("com.twitter.storehaus.testing.Cleanup$")
       c.getMethod("cleanup").invoke(c.getField("MODULE$").get(c))
@@ -28,38 +51,27 @@ object StorehausBuild extends Build {
 
   val sharedSettings = extraSettings ++ ciSettings ++ Seq(
     organization := "com.twitter",
-    crossScalaVersions := Seq("2.9.2", "2.10.0"),
-
+    scalaVersion := "2.9.3",
+    crossScalaVersions := Seq("2.9.3", "2.10.0"),
     javacOptions ++= Seq("-source", "1.6", "-target", "1.6"),
-
     javacOptions in doc := Seq("-source", "1.6"),
-
-    libraryDependencies ++= Seq(
-      "org.scala-tools.testing" %% "specs" % "1.6.9" % "test" withSources()
-    ),
-
+    libraryDependencies += "org.scala-tools.testing" %% "specs" % "1.6.9" % "test" withSources(),
     resolvers ++= Seq(
       Opts.resolver.sonatypeSnapshots,
       Opts.resolver.sonatypeReleases,
       "Twitter Maven" at "http://maven.twttr.com"
     ),
-
     parallelExecution in Test := true,
-
     scalacOptions ++= Seq(Opts.compile.unchecked, Opts.compile.deprecation),
 
     // Publishing options:
     publishMavenStyle := true,
-
     publishArtifact in Test := false,
-
     pomIncludeRepository := { x => false },
-
     publishTo <<= version { v =>
       Some(if (v.trim.toUpperCase.endsWith("SNAPSHOT")) Opts.resolver.sonatypeSnapshots
            else Opts.resolver.sonatypeStaging)
     },
-
     pomExtra := (
       <url>https://github.com/twitter/storehaus</url>
       <licenses>
@@ -97,10 +109,10 @@ object StorehausBuild extends Build {
   def youngestForwardCompatible(subProj: String) =
     Some(subProj)
       .filterNot(unreleasedModules.contains(_))
-      .map { s => "com.twitter" % ("storehaus-" + s + "_2.9.2") % "0.3.0" }
+      .map { s => "com.twitter" % ("storehaus-" + s + "_2.9.3") % "0.5.0" }
 
-  val algebirdVersion = "0.1.13"
-  val bijectionVersion = "0.4.0"
+  val algebirdVersion = "0.2.0"
+  val bijectionVersion = "0.5.3"
 
   lazy val storehaus = Project(
     id = "storehaus",
@@ -117,42 +129,63 @@ object StorehausBuild extends Build {
     storehausMemcache,
     storehausMySQL,
     storehausRedis,
+    storehausHBase,
     storehausTesting
   )
 
   def module(name: String) = {
     val id = "storehaus-%s".format(name)
-    Project(id = id, base = file(id), settings = sharedSettings ++ Seq(
+    Project(id = id, base = file(id), settings = sharedSettings ++ testCleanup ++ Seq(
       Keys.name := id,
-      previousArtifact := youngestForwardCompatible(name)) ++ testCleanup
+      previousArtifact := youngestForwardCompatible(name))
     ).dependsOn(storehausTesting % "test->test")
   }
 
   lazy val storehausCache = module("cache")
 
   lazy val storehausCore = module("core").settings(
-    libraryDependencies += "com.twitter" %% "util-core" % "6.3.0",
-    libraryDependencies += "com.twitter" %% "bijection-core" % bijectionVersion
+    libraryDependencies ++= Seq(
+      withCross("com.twitter" %% "util-core" % "6.3.7"),
+      "com.twitter" %% "bijection-core" % bijectionVersion,
+      "com.twitter" %% "bijection-util" % bijectionVersion
+    )
   ).dependsOn(storehausCache %  "test->test;compile->compile")
 
   lazy val storehausAlgebra = module("algebra").settings(
     libraryDependencies += "com.twitter" %% "algebird-core" % algebirdVersion,
     libraryDependencies += "com.twitter" %% "algebird-util" % algebirdVersion,
-    libraryDependencies += "com.twitter" %% "bijection-algebird" % bijectionVersion
+    libraryDependencies += "com.twitter" %% "algebird-bijection" % algebirdVersion
   ).dependsOn(storehausCore % "test->test;compile->compile")
 
   lazy val storehausMemcache = module("memcache").settings(
-    libraryDependencies += Finagle.module("memcached")
+    libraryDependencies ++= Seq(
+      "com.twitter" %% "algebird-core" % algebirdVersion,
+      "com.twitter" %% "bijection-core" % bijectionVersion,
+      "com.twitter" %% "bijection-netty" % bijectionVersion,
+      Finagle.module("memcached")
+    )
   ).dependsOn(storehausAlgebra % "test->test;compile->compile")
 
   lazy val storehausMySQL = module("mysql").settings(
-    libraryDependencies += Finagle.module("mysql", "6.2.1") // tests fail with the latest
+    libraryDependencies += Finagle.module("mysql")
   ).dependsOn(storehausCore % "test->test;compile->compile")
 
   lazy val storehausRedis = module("redis").settings(
     libraryDependencies += Finagle.module("redis"),
     // we don't want various tests clobbering each others keys
-    parallelExecution in Test := false 
+    parallelExecution in Test := false
+  ).dependsOn(storehausAlgebra % "test->test;compile->compile")
+
+  lazy val storehausHBase= module("hbase").settings(
+    libraryDependencies ++= Seq(
+      "com.twitter" %% "algebird-core" % algebirdVersion,
+      "com.twitter" %% "bijection-core" % bijectionVersion,
+      "com.twitter" %% "bijection-hbase" % bijectionVersion ,
+      "org.apache.hbase" % "hbase" % "0.94.6" % "provided->default" classifier "tests" classifier "",
+      "org.apache.hadoop" % "hadoop-core" % "1.2.0" % "provided->default",
+      "org.apache.hadoop" % "hadoop-test" % "1.2.0" % "test"
+    ),
+    parallelExecution in Test := false
   ).dependsOn(storehausAlgebra % "test->test;compile->compile")
 
   val storehausTesting = Project(
@@ -161,9 +194,7 @@ object StorehausBuild extends Build {
     settings = sharedSettings ++ Seq(
       name := "storehaus-testing",
       previousArtifact := youngestForwardCompatible("testing"),
-      libraryDependencies ++= Seq(
-        "org.scalacheck" %% "scalacheck" % "1.10.0" withSources()
-      )
+      libraryDependencies += "org.scalacheck" %% "scalacheck" % "1.10.0" withSources()
     )
   )
 }

project/Finagle.scala

@@ -5,7 +5,7 @@ package storehaus
  *  dependency */
 object Finagle {
   import sbt._
-  val LatestVersion = "6.3.0"
+  val LatestVersion = "6.5.1"
   def module(name: String, version: String = LatestVersion) =
-    "com.twitter" %% "finagle-%s".format(name) % version
+    StorehausBuild.withCross("com.twitter" %% "finagle-%s".format(name) % version)
 }

project/build.properties

@@ -1 +1 @@
-sbt.version=0.12.0
+sbt.version=0.12.0

storehaus-algebra/src/main/scala/com/twitter/storehaus/algebra/ConvertedMergeableStore.scala

@@ -32,7 +32,7 @@ class ConvertedMergeableStore[K1, -K2, V1, V2](store: MergeableStore[K1, V1])(kf
   (implicit bij: ImplicitBijection[V2, V1])
   extends com.twitter.storehaus.ConvertedStore[K1, K2, V1, V2](store)(kfn)(Injection.fromBijection(bij.bijection))
   with MergeableStore[K2, V2] {
-  import com.twitter.bijection.algebird.AlgebirdBijections._
+  import com.twitter.algebird.bijection.AlgebirdBijections._
 
   override def monoid: Monoid[V2] = store.monoid.as[Monoid[V2]]
 

storehaus-algebra/src/main/scala/com/twitter/storehaus/algebra/MutableCacheAlgebra.scala

@@ -42,21 +42,24 @@ class AlgebraicMutableCache[K, V](cache: MutableCache[K, V]) {
     new MutableCache[K, U] {
       override def get(k: K) = for {
         v <- cache.get(k)
-        (_, u) <- injection.invert(k, v)
+        (_, u) <- injection.invert(k, v).toOption
       } yield u
 
       override def +=(ku: (K, U)) = { cache += injection(ku); this }
 
-      override def hit(k: K) = cache.hit(k).flatMap { injection.invert(k, _) }.map { _._2 }
+      override def hit(k: K) =
+        cache.hit(k)
+          .flatMap(injection.invert(k, _).toOption).map(_._2)
 
       override def evict(k: K) = for {
         evictedV <- cache.evict(k)
-        (_, evictedU) <- injection.invert(k, evictedV)
+        (_, evictedU) <- injection.invert(k, evictedV).toOption
       } yield evictedU
 
       override def empty = new AlgebraicMutableCache(cache.empty).inject(injection)
 
       override def clear = { cache.clear; this }
-      override def iterator = cache.iterator.flatMap(injection.invert(_))
+      override def iterator =
+        cache.iterator.flatMap(injection.invert(_).toOption)
     }
 }

storehaus-algebra/src/main/scala/com/twitter/storehaus/algebra/TTLInjection.scala

@@ -18,6 +18,7 @@ package com.twitter.storehaus.algebra
 
 import com.twitter.algebird.Semigroup
 import com.twitter.bijection.Injection
+import scala.util.{ Success, Failure }
 
 /**
   * Injection that maps values paired with stale values of T => None
@@ -27,17 +28,19 @@ import com.twitter.bijection.Injection
   * @author Sam Ritchie
   */
 
+case class ExpiredException[K, V](pair: (K, V)) extends RuntimeException(pair.toString)
+
 class TTLInjection[K, T: Ordering: Semigroup, V](delta: T)(clock: () => T) extends Injection[(K, V), (K, (T, V))] {
   def apply(pair: (K, V)): (K, (T, V)) = {
     val (k, v) = pair
     (k, (Semigroup.plus(clock(), delta), v))
   }
 
-  override def invert(pair: (K, (T, V))): Option[(K, V)] = {
+  override def invert(pair: (K, (T, V))) = {
     val (k, (expiration, v)) = pair
     if (Ordering[T].gteq(expiration, clock()))
-      Some((k, v))
+      Success(k -> v)
     else
-      None
+      Failure(ExpiredException(k -> v))
   }
 }

storehaus-algebra/src/test/scala/com/twitter/storehaus/algebra/MergeableStoreProperties.scala

@@ -17,7 +17,7 @@
 package com.twitter.storehaus.algebra
 
 import com.twitter.algebird.{ MapAlgebra, Monoid, SummingQueue }
-import com.twitter.bijection.algebird.AlgebirdBijections._
+import com.twitter.algebird.bijection.AlgebirdBijections._
 import com.twitter.bijection.Injection
 import com.twitter.storehaus._
 import com.twitter.util.Await
@@ -93,7 +93,7 @@ object MergeableStoreProperties extends Properties("MergeableStore") {
 
   property("Converted MergeableStore obeys the mergeable store laws") = {
     // We are using a weird monoid on Int here:
-    import com.twitter.bijection.algebird.AlgebirdBijections._
+    import com.twitter.algebird.bijection.AlgebirdBijections._
     import com.twitter.bijection.Conversion.asMethod
     implicit val monoid : Monoid[Int] = implicitly[Monoid[(Short,Short)]].as[Monoid[Int]]
 

storehaus-cache/src/main/scala/com/twitter/storehaus/cache/MutableLRUCache.scala

@@ -31,7 +31,7 @@ object MutableLRUCache {
 }
 
 class MutableLRUCache[K, V](capacity: Int) extends JMapCache[K, V](() =>
-  new JLinkedHashMap[K, V](capacity + 1, 0.75f) {
+  new JLinkedHashMap[K, V](capacity + 1, 0.75f, true) {
     override protected def removeEldestEntry(eldest: JMap.Entry[K, V]) =
       super.size > capacity
   }) {