-
Notifications
You must be signed in to change notification settings - Fork 85
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #139 from MansurAshraf/develop
Added storehaus-hbase
- Loading branch information
Showing
10 changed files
with
493 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
80 changes: 80 additions & 0 deletions
80
storehaus-hbase/src/main/scala/com/twitter/storehaus/hbase/HBaseByteArrayStore.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| /* | ||
| * Copyright 2013 Twitter inc. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package com.twitter.storehaus.hbase | ||
|
|
||
| import org.apache.hadoop.hbase.client.HTablePool | ||
| import com.twitter.storehaus.Store | ||
| import com.twitter.util.Future | ||
| import org.apache.hadoop.conf.Configuration | ||
|
|
||
| /** | ||
| * @author MansurAshraf | ||
| * @since 9/8/13 | ||
| */ | ||
| object HBaseByteArrayStore { | ||
| def apply(quorumNames: Seq[String], | ||
| table: String, | ||
| columnFamily: String, | ||
| column: String, | ||
| createTable: Boolean, | ||
| pool: HTablePool, | ||
| conf: Configuration, | ||
| threads: Int): HBaseByteArrayStore = { | ||
| val store = new HBaseByteArrayStore(quorumNames, table, columnFamily, column, createTable, pool, conf, threads) | ||
| store.validateConfiguration() | ||
| store.createTableIfRequired() | ||
| store | ||
| } | ||
|
|
||
| def apply(quorumNames: Seq[String], | ||
| table: String, | ||
| columnFamily: String, | ||
| column: String, | ||
| createTable: Boolean): HBaseByteArrayStore = apply(quorumNames, table, columnFamily, column, createTable, new HTablePool(), new Configuration(), 4) | ||
| } | ||
|
|
||
| class HBaseByteArrayStore(protected val quorumNames: Seq[String], | ||
| protected val table: String, | ||
| protected val columnFamily: String, | ||
| protected val column: String, | ||
| protected val createTable: Boolean, | ||
| protected val pool: HTablePool, | ||
| protected val conf: Configuration, | ||
| protected val threads: Int) extends Store[Array[Byte], Array[Byte]] with HBaseStore { | ||
|
|
||
| /** get a single key from the store. | ||
| * Prefer multiGet if you are getting more than one key at a time | ||
| */ | ||
| override def get(k: Array[Byte]): Future[Option[Array[Byte]]] = { | ||
| getValue(k) | ||
| } | ||
|
|
||
| /** | ||
| * replace a value | ||
| * Delete is the same as put((k,None)) | ||
| */ | ||
| override def put(kv: (Array[Byte], Option[Array[Byte]])): Future[Unit] = { | ||
| putValue(kv) | ||
| } | ||
|
|
||
| /** Close this store and release any resources. | ||
| * It is undefined what happens on get/multiGet after close | ||
| */ | ||
| override def close { | ||
| pool.close() | ||
| } | ||
| } |
88 changes: 88 additions & 0 deletions
88
storehaus-hbase/src/main/scala/com/twitter/storehaus/hbase/HBaseLongStore.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| /* | ||
| * Copyright 2013 Twitter inc. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package com.twitter.storehaus.hbase | ||
|
|
||
| import org.apache.hadoop.hbase.client.HTablePool | ||
| import com.twitter.storehaus.Store | ||
| import com.twitter.util.Future | ||
| import com.twitter.bijection.Injection._ | ||
| import org.apache.hadoop.conf.Configuration | ||
|
|
||
| /** | ||
| * @author Mansur Ashraf | ||
| * @since 9/8/13 | ||
| */ | ||
| object HBaseLongStore { | ||
| def apply(quorumNames: Seq[String], | ||
| table: String, | ||
| columnFamily: String, | ||
| column: String, | ||
| createTable: Boolean, | ||
| pool: HTablePool, | ||
| conf: Configuration, | ||
| threads:Int): HBaseLongStore = { | ||
| val store = new HBaseLongStore(quorumNames, table, columnFamily, column, createTable, pool, conf,threads) | ||
| store.validateConfiguration() | ||
| store.createTableIfRequired() | ||
| store | ||
| } | ||
|
|
||
| def apply(quorumNames: Seq[String], | ||
| table: String, | ||
| columnFamily: String, | ||
| column: String, | ||
| createTable: Boolean): HBaseLongStore = apply(quorumNames, table, columnFamily, column, createTable, new HTablePool(), new Configuration(),4) | ||
| } | ||
|
|
||
| class HBaseLongStore(protected val quorumNames: Seq[String], | ||
| protected val table: String, | ||
| protected val columnFamily: String, | ||
| protected val column: String, | ||
| protected val createTable: Boolean, | ||
| protected val pool: HTablePool, | ||
| protected val conf: Configuration, | ||
| protected val threads:Int) extends Store[String, Long] with HBaseStore { | ||
|
|
||
| /** get a single key from the store. | ||
| * Prefer multiGet if you are getting more than one key at a time | ||
| */ | ||
| override def get(k: String): Future[Option[Long]] = { | ||
| import com.twitter.bijection.hbase.HBaseBijections._ | ||
| implicit val stringInj = fromBijectionRep[String, StringBytes] | ||
| implicit val LongInj = fromBijectionRep[Long, LongBytes] | ||
| getValue[String, Long](k) | ||
| } | ||
|
|
||
| /** | ||
| * replace a value | ||
| * Delete is the same as put((k,None)) | ||
| */ | ||
| override def put(kv: (String, Option[Long])): Future[Unit] = { | ||
| import com.twitter.bijection.hbase.HBaseBijections._ | ||
| implicit val stringInj = fromBijectionRep[String, StringBytes] | ||
| implicit val LongInj = fromBijectionRep[Long, LongBytes] | ||
| putValue(kv) | ||
| } | ||
|
|
||
| /** Close this store and release any resources. | ||
| * It is undefined what happens on get/multiGet after close | ||
| */ | ||
| override def close { | ||
| pool.close() | ||
| } | ||
| } | ||
|
|
95 changes: 95 additions & 0 deletions
95
storehaus-hbase/src/main/scala/com/twitter/storehaus/hbase/HBaseStore.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| /* | ||
| * Copyright 2013 Twitter inc. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package com.twitter.storehaus.hbase | ||
|
|
||
| import org.apache.hadoop.hbase.client._ | ||
| import org.apache.hadoop.conf.Configuration | ||
| import org.apache.hadoop.hbase.{HColumnDescriptor, HTableDescriptor, HBaseConfiguration} | ||
| import com.twitter.bijection.hbase.HBaseBijections._ | ||
| import com.twitter.bijection.Conversion._ | ||
| import com.twitter.bijection.Injection | ||
| import com.twitter.util.{FuturePool, Future} | ||
| import scala.Some | ||
| import java.util.concurrent.Executors | ||
|
|
||
| /** | ||
| * @author Mansur Ashraf | ||
| * @since 9/8/13 | ||
| */ | ||
| trait HBaseStore { | ||
|
|
||
| protected val quorumNames: Seq[String] | ||
| protected val createTable: Boolean | ||
| protected val table: String | ||
| protected val columnFamily: String | ||
| protected val column: String | ||
| protected val pool: HTablePool | ||
| protected val conf: Configuration | ||
| protected val threads: Int | ||
| protected val futurePool = FuturePool(Executors.newFixedThreadPool(threads)) | ||
|
|
||
| def getHBaseAdmin: HBaseAdmin = { | ||
| if (conf.get("hbase.zookeeper.quorum") == null) { | ||
| conf.set("hbase.zookeeper.quorum", quorumNames.mkString(",")) | ||
| } | ||
| val hbaseConf = HBaseConfiguration.create(conf) | ||
| new HBaseAdmin(hbaseConf) | ||
| } | ||
|
|
||
| def createTableIfRequired() { | ||
| val hbaseAdmin = getHBaseAdmin | ||
| if (createTable && !hbaseAdmin.tableExists(table)) { | ||
| val tableDescriptor = new HTableDescriptor(table) | ||
| tableDescriptor.addFamily(new HColumnDescriptor(columnFamily)) | ||
| hbaseAdmin.createTable(tableDescriptor) | ||
| } | ||
| } | ||
|
|
||
| def validateConfiguration() { | ||
| import org.apache.commons.lang.StringUtils.isNotEmpty | ||
|
|
||
| require(!quorumNames.isEmpty, "Zookeeper quorums are required") | ||
| require(isNotEmpty(columnFamily), "column family is required") | ||
| require(isNotEmpty(column), "column is required") | ||
| } | ||
|
|
||
| def getValue[K, V](key: K)(implicit keyInj: Injection[K, Array[Byte]], valueInj: Injection[V, Array[Byte]]): Future[Option[V]] = futurePool { | ||
| val tbl = pool.getTable(table) | ||
| val g = new Get(keyInj(key)) | ||
| g.addColumn(columnFamily.as[StringBytes], column.as[StringBytes]) | ||
|
|
||
| val result = tbl.get(g) | ||
| val value = result.getValue(columnFamily.as[StringBytes], column.as[StringBytes]) | ||
| Option(value).map(v => valueInj.invert(v).get) | ||
| } | ||
|
|
||
| def putValue[K, V](kv: (K, Option[V]))(implicit keyInj: Injection[K, Array[Byte]], valueInj: Injection[V, Array[Byte]]): Future[Unit] = { | ||
| kv match { | ||
| case (k, Some(v)) => futurePool { | ||
| val p = new Put(keyInj(k)) | ||
| p.add(columnFamily.as[StringBytes], column.as[StringBytes], valueInj(v)) | ||
| val tbl = pool.getTable(table) | ||
| tbl.put(p) | ||
| } | ||
| case (k, None) => futurePool { | ||
| val delete = new Delete(keyInj(k)) | ||
| val tbl = pool.getTable(table) | ||
| tbl.delete(delete) | ||
| } | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.