Skip to content

Commit

Permalink
Fix flakey hbase unit tests (#300)
Browse files Browse the repository at this point in the history
* formatting

* rmd unused imports

* separated the single puts from the multi put properties

* cleanup of DefaultHBaseCluster

* refactored HBaseStringStoreProperties
  • Loading branch information
BenFradet authored and johnynek committed Jun 13, 2016
1 parent f2987e4 commit bcbd9dd
Show file tree
Hide file tree
Showing 7 changed files with 59 additions and 88 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ package com.twitter.storehaus.hbase
import org.apache.hadoop.hbase.client.HTablePool
import com.twitter.storehaus.Store
import com.twitter.util.{Future, Time}
import com.twitter.bijection.Injection._
import org.apache.hadoop.conf.Configuration

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,8 @@ import org.apache.hadoop.hbase.client._
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hbase.{HColumnDescriptor, HTableDescriptor, HBaseConfiguration}
import com.twitter.bijection.hbase.HBaseInjections.string2BytesInj
import com.twitter.bijection.Conversion._
import com.twitter.bijection.Injection
import com.twitter.util.{FuturePool, Future}
import scala.Some
import java.util.concurrent.Executors

/**
Expand Down Expand Up @@ -62,7 +60,7 @@ trait HBaseStore {
def validateConfiguration() {
import org.apache.commons.lang.StringUtils.isNotEmpty

require(!quorumNames.isEmpty, "Zookeeper quorums are required")
require(quorumNames.nonEmpty, "Zookeeper quorums are required")
require(isNotEmpty(columnFamily), "column family is required")
require(isNotEmpty(column), "column is required")
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ package com.twitter.storehaus.hbase
import org.apache.hadoop.hbase.client._
import com.twitter.storehaus.Store
import com.twitter.util.{Future, Time}
import com.twitter.bijection.Injection._
import org.apache.hadoop.conf.Configuration

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,17 +27,20 @@ import com.twitter.util.Closable
*/
trait DefaultHBaseCluster[C <: Closable] extends CloseableCleanup[C] {
val quorumNames = Seq("localhost:2181")
val table = "summing_bird"
val columnFamily = "sb"
val column = "aggregate"
val table = "table"
val columnFamily = "columnFamily"
val column = "column"
val createTable = true
val testingUtil = new HBaseTestingUtility()
val testingUtil = {
val tu = new HBaseTestingUtility()
tu.startMiniCluster()
tu
}
val conf = testingUtil.getConfiguration
val pool = new HTablePool(conf, 1)

override def cleanup() = {
override def cleanup(): Unit = {
super.cleanup()
/* testingUtil.shutdownMiniZKCluster()
testingUtil.shutdownMiniCluster()*/
testingUtil.shutdownMiniCluster()
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Copyright 2014 Twitter inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.twitter.storehaus.hbase

import com.twitter.storehaus.Store
import com.twitter.storehaus.testing.generator.NonEmpty
import com.twitter.util.Await
import org.scalacheck.{Arbitrary, Gen, Properties}
import org.scalacheck.Prop._

/**
* @author Mansur Ashraf
* @since 9/8/13
*/
object HBaseStoreProperties extends Properties("HBaseStore")
with DefaultHBaseCluster[Store[String, String]] {

def putAndGetTest[K: Arbitrary, V: Arbitrary : Equiv](
store: Store[K, V], pairs: Gen[List[(K, Option[V])]]) =
forAll(pairs) { examples =>
examples.forall {
case (k, v) =>
Await.result(store.put((k, v)))
val found = Await.result(store.get(k))
Equiv[Option[V]].equiv(found, v)
}
}

val closeable =
HBaseStringStore(quorumNames, table, columnFamily, column, createTable, pool, conf, 4)
property("HBaseStore[String, String]") =
putAndGetTest(closeable, NonEmpty.Pairing.alphaStrs())
}

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,13 @@ object LevelDBStoreProperties extends Properties("LevelDBStore") {
pairs: Gen[List[(Array[Byte], Option[Array[Byte]])]]) =
forAll(pairs) { examples: List[(Array[Byte], Option[Array[Byte]])] =>
examples.forall {
case (k, v) => {
case (k, v) =>
Await.result(store.put(k, v))
val found = Await.result(store.get(k))
found match {
case Some(a) => util.Arrays.equals(a, v.get)
case None => found == v
}
}
}
}

Expand Down

0 comments on commit bcbd9dd

Please sign in to comment.