From cc63ebe22f357d059f29b64051169bcc77e64dd2 Mon Sep 17 00:00:00 2001 From: maryannxue Date: Wed, 31 Jul 2013 16:56:15 -0400 Subject: [PATCH 001/102] remove unused methods in HashCache --- .../salesforce/phoenix/cache/HashCache.java | 2 -- .../phoenix/cache/TenantCacheImpl.java | 26 ------------------- .../phoenix/join/HashCacheClient.java | 13 +++------- 3 files changed, 3 insertions(+), 38 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/cache/HashCache.java b/src/main/java/com/salesforce/phoenix/cache/HashCache.java index 8dadc2c7..d4e89010 100644 --- a/src/main/java/com/salesforce/phoenix/cache/HashCache.java +++ b/src/main/java/com/salesforce/phoenix/cache/HashCache.java @@ -44,7 +44,5 @@ */ @Immutable public interface HashCache extends SQLCloseable { - public byte[] getTableName(); - public byte[][] getColumnFamilies(); public List get(ImmutableBytesWritable hashKey); } diff --git a/src/main/java/com/salesforce/phoenix/cache/TenantCacheImpl.java b/src/main/java/com/salesforce/phoenix/cache/TenantCacheImpl.java index c72a4b20..0cb8ce39 100644 --- a/src/main/java/com/salesforce/phoenix/cache/TenantCacheImpl.java +++ b/src/main/java/com/salesforce/phoenix/cache/TenantCacheImpl.java @@ -56,7 +56,6 @@ import com.salesforce.phoenix.memory.MemoryManager.MemoryChunk; import com.salesforce.phoenix.schema.tuple.ResultTuple; import com.salesforce.phoenix.schema.tuple.Tuple; -import com.salesforce.phoenix.util.ByteUtil; import com.salesforce.phoenix.util.ImmutableBytesPtr; import com.salesforce.phoenix.util.SQLCloseable; import com.salesforce.phoenix.util.SizedUtil; @@ -181,8 +180,6 @@ public void close() throws SQLException { private class AgeOutHashCache implements HashCache { private final Map> hashCache; private final MemoryChunk memoryChunk; - private final byte[][] cfs; - private final byte[] tableName; private AgeOutHashCache(ImmutableBytesWritable hashCacheBytes) { try { @@ -220,19 +217,6 @@ private AgeOutHashCache(ImmutableBytesWritable hashCacheBytes) { tuples.add(result); offset += resultSize; } - int cfCount = (int)Bytes.readVLong(hashCacheByteArray, offset); - if (cfCount == 0) { - cfs = null; - tableName = null; - } else { - offset += WritableUtils.decodeVIntSize(hashCacheByteArray[offset]); - byte[][] tableInfo = ByteUtil.toByteArrays(hashCacheByteArray, offset, cfCount); - tableName = tableInfo[0]; - cfs = new byte[cfCount - 1][]; - for (int i = 1; i < cfCount; i++) { - cfs[i-1] = tableInfo[i]; - } - } this.hashCache = Collections.unmodifiableMap(hashCacheMap); } catch (IOException e) { // Not possible with ByteArrayInputStream throw new RuntimeException(e); @@ -243,16 +227,6 @@ private AgeOutHashCache(ImmutableBytesWritable hashCacheBytes) { public void close() { memoryChunk.close(); } - - @Override - public byte[] getTableName() { - return tableName; - } - - @Override - public byte[][] getColumnFamilies() { - return cfs; - } @Override public List get(ImmutableBytesWritable hashKey) { diff --git a/src/main/java/com/salesforce/phoenix/join/HashCacheClient.java b/src/main/java/com/salesforce/phoenix/join/HashCacheClient.java index 8a33c647..6863d652 100644 --- a/src/main/java/com/salesforce/phoenix/join/HashCacheClient.java +++ b/src/main/java/com/salesforce/phoenix/join/HashCacheClient.java @@ -144,7 +144,7 @@ public void close() throws SQLException { * @throws MaxHashCacheSizeExceededException if size of hash cache exceeds max allowed * size */ - public HashCache addHashCache(Scanner scanner, List onExpressions, byte[] tableName, byte[][] cfs) throws SQLException { + public HashCache addHashCache(Scanner scanner, List onExpressions) throws SQLException { final byte[] joinId = nextJoinId(); /** @@ -160,7 +160,7 @@ public HashCache addHashCache(Scanner scanner, List onExpressions, b closeables.add(chunk); try { iterator = scanner.iterator(); - hashCache = serialize(iterator, onExpressions, tableName, cfs, chunk); + hashCache = serialize(iterator, onExpressions, chunk); } finally { if (iterator != null) { iterator.close(); @@ -287,7 +287,7 @@ private static synchronized byte[] nextJoinId() { } // package private for testing - ImmutableBytesWritable serialize(ResultIterator scanner, List onExpressions, byte[] tableName, byte[][] cfs, MemoryChunk chunk) throws SQLException { + ImmutableBytesWritable serialize(ResultIterator scanner, List onExpressions, MemoryChunk chunk) throws SQLException { try { long maxSize = services.getConfig().getLong(QueryServices.MAX_HASH_CACHE_SIZE_ATTRIB, DEFAULT_MAX_HASH_CACHE_SIZE); long estimatedSize = Math.min(chunk.getSize(), maxSize); @@ -317,13 +317,6 @@ ImmutableBytesWritable serialize(ResultIterator scanner, List onExpr } nRows++; } - if (cfs == null) { - WritableUtils.writeVInt(out, 0); - } else { - WritableUtils.writeVInt(out, cfs.length + 1); - Bytes.writeByteArray(out, tableName); - out.write(ByteUtil.toBytes(cfs)); - } TrustedByteArrayOutputStream sizeOut = new TrustedByteArrayOutputStream(Bytes.SIZEOF_INT); DataOutputStream dataOut = new DataOutputStream(sizeOut); try { From 967a782e9ffdd830341b97015b603acc9f50297e Mon Sep 17 00:00:00 2001 From: maryannxue Date: Thu, 1 Aug 2013 17:02:51 -0400 Subject: [PATCH 002/102] add JoinStatement and related classes --- .../phoenix/compile/JoinCompiler.java | 32 ++++++++++++ .../phoenix/parse/JoinStatement.java | 50 ++++++++++++++++++ .../phoenix/parse/JoinTableNode.java | 12 ++--- .../phoenix/parse/ParseNodeFactory.java | 8 ++- .../phoenix/parse/SimpleJoinTableNode.java | 51 +++++++++++++++++++ .../phoenix/parse/SubqueryJoinTableNode.java | 24 +++++++++ 6 files changed, 169 insertions(+), 8 deletions(-) create mode 100644 src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java create mode 100644 src/main/java/com/salesforce/phoenix/parse/JoinStatement.java create mode 100644 src/main/java/com/salesforce/phoenix/parse/SimpleJoinTableNode.java create mode 100644 src/main/java/com/salesforce/phoenix/parse/SubqueryJoinTableNode.java diff --git a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java new file mode 100644 index 00000000..0d2846f3 --- /dev/null +++ b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java @@ -0,0 +1,32 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.phoenix.compile; + + +public class JoinCompiler { +} diff --git a/src/main/java/com/salesforce/phoenix/parse/JoinStatement.java b/src/main/java/com/salesforce/phoenix/parse/JoinStatement.java new file mode 100644 index 00000000..fd22175d --- /dev/null +++ b/src/main/java/com/salesforce/phoenix/parse/JoinStatement.java @@ -0,0 +1,50 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.phoenix.parse; + +import java.util.List; + +public class JoinStatement extends SelectStatement { + + private List joinTables; + + protected JoinStatement(List from, HintNode hint, + boolean isDistinct, List select, ParseNode where, + List groupBy, ParseNode having, + List orderBy, LimitNode limit, int bindCount, + List joinTables) { + super(from, hint, isDistinct, select, where, groupBy, having, orderBy, limit, + bindCount); + this.joinTables = joinTables; + } + + public List getJoinTables() { + return joinTables; + } + +} diff --git a/src/main/java/com/salesforce/phoenix/parse/JoinTableNode.java b/src/main/java/com/salesforce/phoenix/parse/JoinTableNode.java index 7f456068..b4042728 100644 --- a/src/main/java/com/salesforce/phoenix/parse/JoinTableNode.java +++ b/src/main/java/com/salesforce/phoenix/parse/JoinTableNode.java @@ -38,24 +38,24 @@ * @author jtaylor * @since 0.1 */ -public class JoinTableNode extends TableNode { +public abstract class JoinTableNode extends TableNode { public enum JoinType {Inner, Left, Right, Full}; - private final NamedTableNode table; private final JoinType type; + private final ParseNode on; - JoinTableNode(String alias, NamedTableNode table, ParseNode node, JoinType type) { + JoinTableNode(String alias, JoinType type, ParseNode on) { super(alias); - this.table = table; this.type = type; + this.on = on; } public JoinType getType() { return type; } - public NamedTableNode getTable() { - return table; + public ParseNode getOnNode() { + return on; } @Override diff --git a/src/main/java/com/salesforce/phoenix/parse/ParseNodeFactory.java b/src/main/java/com/salesforce/phoenix/parse/ParseNodeFactory.java index 042b117e..69cc0d99 100644 --- a/src/main/java/com/salesforce/phoenix/parse/ParseNodeFactory.java +++ b/src/main/java/com/salesforce/phoenix/parse/ParseNodeFactory.java @@ -343,8 +343,12 @@ public IsNullParseNode isNull(ParseNode child, boolean negate) { return new IsNullParseNode(child, negate); } - public JoinTableNode join (String alias, NamedTableNode table, ParseNode on, JoinType type) { - return new JoinTableNode(alias, table, on, type); + public SimpleJoinTableNode join (String alias, JoinType type, ParseNode on, NamedTableNode table) { + return new SimpleJoinTableNode(alias, type, on, table); + } + + public SubqueryJoinTableNode join (String alias, JoinType type, ParseNode on, SelectStatement select) { + return new SubqueryJoinTableNode(alias, type, on, select); } public DerivedTableNode subselect (String alias, SelectStatement select) { diff --git a/src/main/java/com/salesforce/phoenix/parse/SimpleJoinTableNode.java b/src/main/java/com/salesforce/phoenix/parse/SimpleJoinTableNode.java new file mode 100644 index 00000000..fd3dc95d --- /dev/null +++ b/src/main/java/com/salesforce/phoenix/parse/SimpleJoinTableNode.java @@ -0,0 +1,51 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.phoenix.parse; + +import java.sql.SQLException; + +public class SimpleJoinTableNode extends JoinTableNode { + + private final NamedTableNode table; + + SimpleJoinTableNode(String alias, JoinType type, ParseNode on, + NamedTableNode table) { + super(alias, type, on); + this.table = table; + } + + public NamedTableNode getTable() { + return table; + } + + @Override + public void accept(TableNodeVisitor visitor) throws SQLException { + visitor.visit(this); + } + +} diff --git a/src/main/java/com/salesforce/phoenix/parse/SubqueryJoinTableNode.java b/src/main/java/com/salesforce/phoenix/parse/SubqueryJoinTableNode.java new file mode 100644 index 00000000..cec5e770 --- /dev/null +++ b/src/main/java/com/salesforce/phoenix/parse/SubqueryJoinTableNode.java @@ -0,0 +1,24 @@ +package com.salesforce.phoenix.parse; + +import java.sql.SQLException; + +public class SubqueryJoinTableNode extends JoinTableNode { + + private final SelectStatement select; + + SubqueryJoinTableNode(String alias, JoinType type, ParseNode on, + SelectStatement select) { + super(alias, type, on); + this.select = select; + } + + public SelectStatement getSelectNode() { + return select; + } + + @Override + public void accept(TableNodeVisitor visitor) throws SQLException { + visitor.visit(this); + } + +} From a2047fda3ae4d62eca616456620456d3bad57c3d Mon Sep 17 00:00:00 2001 From: maryannxue Date: Thu, 1 Aug 2013 18:29:24 -0400 Subject: [PATCH 003/102] add comments for SubqueryJoinTableNode --- .../salesforce/phoenix/parse/SubqueryJoinTableNode.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/com/salesforce/phoenix/parse/SubqueryJoinTableNode.java b/src/main/java/com/salesforce/phoenix/parse/SubqueryJoinTableNode.java index cec5e770..d2622d38 100644 --- a/src/main/java/com/salesforce/phoenix/parse/SubqueryJoinTableNode.java +++ b/src/main/java/com/salesforce/phoenix/parse/SubqueryJoinTableNode.java @@ -2,6 +2,14 @@ import java.sql.SQLException; +/** + * TableNode representing a subquery used as a join table. + * Sub-joins like join (B join C on ...) on ... is be considered a shorthand + * for subqueries, thus will also be represented by this node. + * + * @author wxue3 + * + */ public class SubqueryJoinTableNode extends JoinTableNode { private final SelectStatement select; From 6f6773ba9a68bfc98f35bcba4e71c3d1957c878c Mon Sep 17 00:00:00 2001 From: maryannxue Date: Thu, 1 Aug 2013 18:49:11 -0400 Subject: [PATCH 004/102] modify SQL lexer for joins --- src/main/antlr3/PhoenixSQL.g | 2 +- .../salesforce/phoenix/coprocessor/HashJoinRegionScanner.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/antlr3/PhoenixSQL.g b/src/main/antlr3/PhoenixSQL.g index 659fc7f3..b673cad3 100644 --- a/src/main/antlr3/PhoenixSQL.g +++ b/src/main/antlr3/PhoenixSQL.g @@ -530,7 +530,7 @@ join_specs returns [List ret] ; join_spec returns [JoinTableNode ret] - : j=join_type JOIN t=named_table ON e=condition { $ret = factory.join(null, t, e, j); } + : j=join_type JOIN t=named_table ON e=condition { $ret = factory.join(null, j, e, t); } ; join_type returns [JoinTableNode.JoinType ret] diff --git a/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java b/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java index 3e7a3d5e..b1c2f197 100644 --- a/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java +++ b/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java @@ -67,8 +67,8 @@ public HashJoinRegionScanner(RegionScanner scanner, ScanProjector projector, Has if (tenantId == null) throw new IOException("Could not find tenant id for hash cache."); for (JoinType type : joinInfo.getJoinTypes()) { - if (type == JoinType.Right) - throw new IOException("The hashed table should not be LHS."); + if (type != JoinType.Inner && type != JoinType.Left) + throw new IOException("Got join type '" + type + "'. Expect only INNER or LEFT with hash-joins."); } this.cache = GlobalCache.getTenantCache(conf, tenantId); } From 4463545202c1d499dea909c3c599850143b7d26c Mon Sep 17 00:00:00 2001 From: maryannxue Date: Fri, 2 Aug 2013 14:31:41 -0400 Subject: [PATCH 005/102] modify join compilation --- src/main/antlr3/PhoenixSQL.g | 25 +++------ .../phoenix/parse/JoinStatement.java | 50 ------------------ .../phoenix/parse/JoinTableNode.java | 12 +++-- .../phoenix/parse/ParseNodeFactory.java | 8 +-- .../phoenix/parse/SimpleJoinTableNode.java | 51 ------------------- .../phoenix/parse/SubqueryJoinTableNode.java | 32 ------------ 6 files changed, 18 insertions(+), 160 deletions(-) delete mode 100644 src/main/java/com/salesforce/phoenix/parse/JoinStatement.java delete mode 100644 src/main/java/com/salesforce/phoenix/parse/SimpleJoinTableNode.java delete mode 100644 src/main/java/com/salesforce/phoenix/parse/SubqueryJoinTableNode.java diff --git a/src/main/antlr3/PhoenixSQL.g b/src/main/antlr3/PhoenixSQL.g index b673cad3..45a0f92d 100644 --- a/src/main/antlr3/PhoenixSQL.g +++ b/src/main/antlr3/PhoenixSQL.g @@ -502,22 +502,15 @@ parseOrderByField returns [OrderByNode ret] ; parseFrom returns [List ret] - : l=table_refs { $ret = l; } - | l=join_specs { $ret = l; } - ; - -table_refs returns [List ret] @init{ret = new ArrayList(4); } - : t=table_ref {$ret.add(t);} - (COMMA t=table_ref {$ret.add(t);} )* + : t=table_ref {$ret.add(t);} (s=sub_table_ref { $ret.add(s); })* ; - -// parse a field, if it might be a bind name. -named_table returns [NamedTableNode ret] - : t=from_table_name { $ret = factory.namedTable(null,t,null); } + +sub_table_ref returns [TableNode ret] + : COMMA t=table_ref { $ret = t; } + | t=join_spec { $ret = t; } ; - table_ref returns [TableNode ret] : n=bind_name ((AS)? alias=identifier)? { $ret = factory.bindTable(alias, factory.table(null,n)); } // TODO: review | t=from_table_name ((AS)? alias=identifier)? (LPAREN cdefs=column_defs RPAREN)? { $ret = factory.namedTable(alias, t,cdefs); } @@ -525,12 +518,8 @@ table_ref returns [TableNode ret] ; catch[SQLException e]{throw new RecognitionException();} -join_specs returns [List ret] - : t=named_table {$ret.add(t);} (s=join_spec { $ret.add(s); })+ - ; - -join_spec returns [JoinTableNode ret] - : j=join_type JOIN t=named_table ON e=condition { $ret = factory.join(null, j, e, t); } +join_spec returns [TableNode ret] + : j=join_type JOIN t=table_ref ON e=condition { $ret = factory.join(j, e, t); } ; join_type returns [JoinTableNode.JoinType ret] diff --git a/src/main/java/com/salesforce/phoenix/parse/JoinStatement.java b/src/main/java/com/salesforce/phoenix/parse/JoinStatement.java deleted file mode 100644 index fd22175d..00000000 --- a/src/main/java/com/salesforce/phoenix/parse/JoinStatement.java +++ /dev/null @@ -1,50 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2013, Salesforce.com, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of Salesforce.com nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ******************************************************************************/ -package com.salesforce.phoenix.parse; - -import java.util.List; - -public class JoinStatement extends SelectStatement { - - private List joinTables; - - protected JoinStatement(List from, HintNode hint, - boolean isDistinct, List select, ParseNode where, - List groupBy, ParseNode having, - List orderBy, LimitNode limit, int bindCount, - List joinTables) { - super(from, hint, isDistinct, select, where, groupBy, having, orderBy, limit, - bindCount); - this.joinTables = joinTables; - } - - public List getJoinTables() { - return joinTables; - } - -} diff --git a/src/main/java/com/salesforce/phoenix/parse/JoinTableNode.java b/src/main/java/com/salesforce/phoenix/parse/JoinTableNode.java index b4042728..32834ca3 100644 --- a/src/main/java/com/salesforce/phoenix/parse/JoinTableNode.java +++ b/src/main/java/com/salesforce/phoenix/parse/JoinTableNode.java @@ -38,16 +38,18 @@ * @author jtaylor * @since 0.1 */ -public abstract class JoinTableNode extends TableNode { +public class JoinTableNode extends TableNode { public enum JoinType {Inner, Left, Right, Full}; private final JoinType type; private final ParseNode on; + private final TableNode table; - JoinTableNode(String alias, JoinType type, ParseNode on) { - super(alias); + JoinTableNode(JoinType type, ParseNode on, TableNode table) { + super(table.getAlias()); this.type = type; this.on = on; + this.table = table; } public JoinType getType() { @@ -57,6 +59,10 @@ public JoinType getType() { public ParseNode getOnNode() { return on; } + + public TableNode getTable() { + return table; + } @Override public void accept(TableNodeVisitor visitor) throws SQLException { diff --git a/src/main/java/com/salesforce/phoenix/parse/ParseNodeFactory.java b/src/main/java/com/salesforce/phoenix/parse/ParseNodeFactory.java index 69cc0d99..6b9d52e9 100644 --- a/src/main/java/com/salesforce/phoenix/parse/ParseNodeFactory.java +++ b/src/main/java/com/salesforce/phoenix/parse/ParseNodeFactory.java @@ -343,12 +343,8 @@ public IsNullParseNode isNull(ParseNode child, boolean negate) { return new IsNullParseNode(child, negate); } - public SimpleJoinTableNode join (String alias, JoinType type, ParseNode on, NamedTableNode table) { - return new SimpleJoinTableNode(alias, type, on, table); - } - - public SubqueryJoinTableNode join (String alias, JoinType type, ParseNode on, SelectStatement select) { - return new SubqueryJoinTableNode(alias, type, on, select); + public JoinTableNode join (JoinType type, ParseNode on, TableNode table) { + return new JoinTableNode(type, on, table); } public DerivedTableNode subselect (String alias, SelectStatement select) { diff --git a/src/main/java/com/salesforce/phoenix/parse/SimpleJoinTableNode.java b/src/main/java/com/salesforce/phoenix/parse/SimpleJoinTableNode.java deleted file mode 100644 index fd3dc95d..00000000 --- a/src/main/java/com/salesforce/phoenix/parse/SimpleJoinTableNode.java +++ /dev/null @@ -1,51 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2013, Salesforce.com, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of Salesforce.com nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ******************************************************************************/ -package com.salesforce.phoenix.parse; - -import java.sql.SQLException; - -public class SimpleJoinTableNode extends JoinTableNode { - - private final NamedTableNode table; - - SimpleJoinTableNode(String alias, JoinType type, ParseNode on, - NamedTableNode table) { - super(alias, type, on); - this.table = table; - } - - public NamedTableNode getTable() { - return table; - } - - @Override - public void accept(TableNodeVisitor visitor) throws SQLException { - visitor.visit(this); - } - -} diff --git a/src/main/java/com/salesforce/phoenix/parse/SubqueryJoinTableNode.java b/src/main/java/com/salesforce/phoenix/parse/SubqueryJoinTableNode.java deleted file mode 100644 index d2622d38..00000000 --- a/src/main/java/com/salesforce/phoenix/parse/SubqueryJoinTableNode.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.salesforce.phoenix.parse; - -import java.sql.SQLException; - -/** - * TableNode representing a subquery used as a join table. - * Sub-joins like join (B join C on ...) on ... is be considered a shorthand - * for subqueries, thus will also be represented by this node. - * - * @author wxue3 - * - */ -public class SubqueryJoinTableNode extends JoinTableNode { - - private final SelectStatement select; - - SubqueryJoinTableNode(String alias, JoinType type, ParseNode on, - SelectStatement select) { - super(alias, type, on); - this.select = select; - } - - public SelectStatement getSelectNode() { - return select; - } - - @Override - public void accept(TableNodeVisitor visitor) throws SQLException { - visitor.visit(this); - } - -} From 7f8f68c2beff4d3a9f6a4db205d805ba627b0c63 Mon Sep 17 00:00:00 2001 From: maryannxue Date: Mon, 5 Aug 2013 23:34:43 -0400 Subject: [PATCH 006/102] add HashJoinPlan --- .../phoenix/compile/JoinCompiler.java | 67 +++++++++++++++ .../phoenix/compile/QueryCompiler.java | 30 ++++++- .../phoenix/compile/StatementContext.java | 11 +++ .../phoenix/execute/HashJoinPlan.java | 81 +++++++++++++++++++ .../phoenix/join/HashCacheClient.java | 18 +++-- 5 files changed, 198 insertions(+), 9 deletions(-) create mode 100644 src/main/java/com/salesforce/phoenix/execute/HashJoinPlan.java diff --git a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java index 0d2846f3..7c4cc65a 100644 --- a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java @@ -27,6 +27,73 @@ ******************************************************************************/ package com.salesforce.phoenix.compile; +import java.sql.SQLException; +import java.util.List; + +import com.salesforce.phoenix.expression.Expression; +import com.salesforce.phoenix.jdbc.PhoenixConnection; +import com.salesforce.phoenix.parse.AliasedNode; +import com.salesforce.phoenix.parse.SelectStatement; +import com.salesforce.phoenix.parse.JoinTableNode.JoinType; +import com.salesforce.phoenix.schema.TableRef; + public class JoinCompiler { + + public enum StarJoinType { + BASIC, + EXTENDED, + NONE, + } + + public static class JoinSpec { + private TableRef table; + private List select; + private List filters; + private List postFilters; + private List joinTables; + private boolean isPostAggregate; + + private JoinSpec(TableRef table, List select, List filters, + List postFilters, List joinTables, boolean isPostAggregate) { + this.table = table; + this.select = select; + this.filters = filters; + this.postFilters = postFilters; + this.joinTables = joinTables; + this.isPostAggregate = isPostAggregate; + } + + public List getJoinTables() { + return joinTables; + } + } + + public static JoinSpec getSubJoinSpec(JoinSpec join) { + return new JoinSpec(join.table, join.select, join.filters, join.postFilters, join.joinTables.subList(0, join.joinTables.size() - 2), join.isPostAggregate); + } + + public static class JoinTable { + private JoinType type; + private List conditions; + private List select; + private List filters; + private List postJoinFilters; // will be pushed to postFilters in case of star join + private TableRef table; + private SelectStatement subquery; + } + + public interface JoinedColumnResolver extends ColumnResolver { + public JoinSpec getJoinTables(); + } + + public static JoinedColumnResolver getResolver(SelectStatement statement, PhoenixConnection connection) throws SQLException { + //TODO + return null; + } + + public static StarJoinType getStarJoinType(JoinSpec join) { + // TODO + return StarJoinType.NONE; + } } diff --git a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java index bb0316d5..75f8e88e 100644 --- a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java @@ -35,12 +35,17 @@ import org.apache.hadoop.hbase.client.Scan; import com.salesforce.phoenix.compile.GroupByCompiler.GroupBy; +import com.salesforce.phoenix.compile.JoinCompiler.JoinSpec; +import com.salesforce.phoenix.compile.JoinCompiler.JoinTable; +import com.salesforce.phoenix.compile.JoinCompiler.JoinedColumnResolver; +import com.salesforce.phoenix.compile.JoinCompiler.StarJoinType; import com.salesforce.phoenix.compile.OrderByCompiler.OrderBy; import com.salesforce.phoenix.execute.AggregatePlan; import com.salesforce.phoenix.execute.ScanPlan; import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.jdbc.PhoenixConnection; import com.salesforce.phoenix.jdbc.PhoenixDatabaseMetaData; +import com.salesforce.phoenix.join.HashCacheClient; import com.salesforce.phoenix.parse.*; import com.salesforce.phoenix.query.QueryConstants; import com.salesforce.phoenix.schema.*; @@ -106,8 +111,29 @@ public QueryPlan compile(SelectStatement statement, List binds) throws S assert(binds.size() == statement.getBindCount()); statement = RHSLiteralStatementRewriter.normalize(statement); - ColumnResolver resolver = FromCompiler.getResolver(statement, connection); - StatementContext context = new StatementContext(connection, resolver, binds, statement.getBindCount(), scan, statement.getHint()); + + List fromNodes = statement.getFrom(); + if (fromNodes.size() == 1) { + ColumnResolver resolver = FromCompiler.getResolver(statement, connection); + StatementContext context = new StatementContext(connection, resolver, binds, statement.getBindCount(), scan, statement.getHint()); + return compile(context, statement, binds); + } + + JoinedColumnResolver resolver = JoinCompiler.getResolver(statement, connection); + StatementContext context = new StatementContext(connection, resolver, binds, statement.getBindCount(), scan, statement.getHint(), new HashCacheClient(connection.getQueryServices(), connection.getTenantId())); + return compile(context, statement, binds); + } + + protected QueryPlan compile(StatementContext context, SelectStatement statement, List binds, JoinSpec join) { + StarJoinType starJoin = JoinCompiler.getStarJoinType(join); + if (starJoin == StarJoinType.BASIC || starJoin == StarJoinType.EXTENDED) { + for (JoinTable joinTable : join.getJoinTables()) { + } + } + } + + protected QueryPlan compile(StatementContext context, SelectStatement statement, List binds) throws SQLException{ + ColumnResolver resolver = context.getResolver(); Map aliasParseNodeMap = ProjectionCompiler.buildAliasParseNodeMap(context, statement.getSelect()); Integer limit = LimitCompiler.getLimit(context, statement.getLimit()); diff --git a/src/main/java/com/salesforce/phoenix/compile/StatementContext.java b/src/main/java/com/salesforce/phoenix/compile/StatementContext.java index 24133ca0..e259f368 100644 --- a/src/main/java/com/salesforce/phoenix/compile/StatementContext.java +++ b/src/main/java/com/salesforce/phoenix/compile/StatementContext.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import com.salesforce.phoenix.jdbc.PhoenixConnection; +import com.salesforce.phoenix.join.HashCacheClient; import com.salesforce.phoenix.parse.HintNode; import com.salesforce.phoenix.parse.HintNode.Hint; import com.salesforce.phoenix.query.QueryConstants; @@ -66,6 +67,7 @@ public class StatementContext { private final ImmutableBytesWritable tempPtr; private final PhoenixConnection connection; private final HintNode hintNode; + private final HashCacheClient hashClient; private boolean isAggregate; private long currentTime = QueryConstants.UNSET_TIMESTAMP; @@ -76,6 +78,10 @@ public StatementContext(PhoenixConnection connection, ColumnResolver resolver, L } public StatementContext(PhoenixConnection connection, ColumnResolver resolver, List binds, int bindCount, Scan scan, HintNode hintNode) { + this(connection, resolver, binds, bindCount, scan, hintNode, null); + } + + public StatementContext(PhoenixConnection connection, ColumnResolver resolver, List binds, int bindCount, Scan scan, HintNode hintNode, HashCacheClient hashClient) { this.connection = connection; this.resolver = resolver; this.scan = scan; @@ -88,6 +94,7 @@ public StatementContext(PhoenixConnection connection, ColumnResolver resolver, L this.numberFormat = connection.getQueryServices().getConfig().get(QueryServices.NUMBER_FORMAT_ATTRIB, NumberUtil.DEFAULT_NUMBER_FORMAT); this.tempPtr = new ImmutableBytesWritable(); this.hintNode = hintNode; + this.hashClient = hashClient; } public boolean hasHint(Hint hint) { @@ -117,6 +124,10 @@ public Scan getScan() { public BindManager getBindManager() { return binds; } + + public HashCacheClient getHashClient() { + return hashClient; + } public AggregationManager getAggregationManager() { return aggregates; diff --git a/src/main/java/com/salesforce/phoenix/execute/HashJoinPlan.java b/src/main/java/com/salesforce/phoenix/execute/HashJoinPlan.java new file mode 100644 index 00000000..d68a8e89 --- /dev/null +++ b/src/main/java/com/salesforce/phoenix/execute/HashJoinPlan.java @@ -0,0 +1,81 @@ +package com.salesforce.phoenix.execute; + +import java.sql.ParameterMetaData; +import java.sql.SQLException; +import java.util.List; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; + +import com.salesforce.phoenix.compile.ExplainPlan; +import com.salesforce.phoenix.compile.QueryPlan; +import com.salesforce.phoenix.compile.RowProjector; +import com.salesforce.phoenix.compile.OrderByCompiler.OrderBy; +import com.salesforce.phoenix.expression.Expression; +import com.salesforce.phoenix.join.HashCacheClient; +import com.salesforce.phoenix.join.HashJoinInfo; +import com.salesforce.phoenix.query.KeyRange; +import com.salesforce.phoenix.query.Scanner; +import com.salesforce.phoenix.schema.TableRef; + +public class HashJoinPlan implements QueryPlan { + + private BasicQueryPlan plan; + private HashJoinInfo joinInfo; + private List hashPlans; + + @Override + public Integer getLimit() { + return plan.getLimit(); + } + + @Override + public OrderBy getOrderBy() { + return plan.getOrderBy(); + } + + @Override + public RowProjector getProjector() { + return plan.getProjector(); + } + + @Override + public Scanner getScanner() throws SQLException { + ImmutableBytesWritable[] joinIds = joinInfo.getJoinIds(); + List[] joinExpressions = joinInfo.getJoinExpressions(); + + assert (joinIds.length == joinExpressions.length && joinIds.length == hashPlans.size()); + + HashCacheClient hashClient = plan.getContext().getHashClient(); + // TODO replace with Future execution + for (int i = 0; i < joinIds.length; i++) { + hashClient.addHashCache(joinIds[i].get(), hashPlans.get(i).getScanner(), joinExpressions[i], plan.getTable().getTableName()); + } + return plan.getScanner(); + } + + @Override + public List getSplits() { + return plan.getSplits(); + } + + @Override + public TableRef getTable() { + return plan.getTable(); + } + + @Override + public boolean isAggregate() { + return plan.isAggregate(); + } + + @Override + public ExplainPlan getExplainPlan() throws SQLException { + return plan.getExplainPlan(); + } + + @Override + public ParameterMetaData getParameterMetaData() { + return plan.getParameterMetaData(); + } + +} diff --git a/src/main/java/com/salesforce/phoenix/join/HashCacheClient.java b/src/main/java/com/salesforce/phoenix/join/HashCacheClient.java index 6863d652..b34af52a 100644 --- a/src/main/java/com/salesforce/phoenix/join/HashCacheClient.java +++ b/src/main/java/com/salesforce/phoenix/join/HashCacheClient.java @@ -69,9 +69,9 @@ public class HashCacheClient { private static final Log LOG = LogFactory.getLog(HashCacheClient.class); private static final String JOIN_KEY_PREFIX = "joinKey"; - private final byte[] iterateOverTableName; private final byte[] tenantId; private final ConnectionQueryServices services; + private final Map tableNameMap = new HashMap(); /** * Construct client used to create a serialized cached snapshot of a table and send it to each region server @@ -80,9 +80,8 @@ public class HashCacheClient { * @param iterateOverTableName table name * @param tenantId the tenantId or null if not applicable */ - public HashCacheClient(ConnectionQueryServices services, byte[] iterateOverTableName, byte[] tenantId) { + public HashCacheClient(ConnectionQueryServices services, byte[] tenantId) { this.services = services; - this.iterateOverTableName = iterateOverTableName; this.tenantId = tenantId; } @@ -144,9 +143,8 @@ public void close() throws SQLException { * @throws MaxHashCacheSizeExceededException if size of hash cache exceeds max allowed * size */ - public HashCache addHashCache(Scanner scanner, List onExpressions) throws SQLException { - final byte[] joinId = nextJoinId(); - + public HashCache addHashCache(final byte[] joinId, Scanner scanner, List onExpressions, byte[] iterateOverTableName) throws SQLException { + tableNameMap.put(Bytes.mapKey(joinId), iterateOverTableName); /** * Serialize and compress hashCacheTable */ @@ -253,6 +251,11 @@ public Object getJobId() { */ private void removeHashCache(byte[] joinId, Set servers) throws SQLException { Throwable lastThrowable = null; + byte[] iterateOverTableName = tableNameMap.get(Bytes.mapKey(joinId)); + if (iterateOverTableName == null) { + LOG.warn("The joinId '" + Bytes.toString(joinId) + "' does not exist. Can't remove hash cache."); + return; + } HTableInterface iterateOverTable = services.getTable(iterateOverTableName); NavigableMap locations; try { @@ -277,12 +280,13 @@ private void removeHashCache(byte[] joinId, Set servers) throws SQLE if (!remainingOnServers.isEmpty()) { LOG.warn("Unable to remove hash cache for " + remainingOnServers, lastThrowable); } + tableNameMap.remove(Bytes.mapKey(joinId)); } /** * Create a join ID to keep the cached information across other joins independent. */ - private static synchronized byte[] nextJoinId() { + public static synchronized byte[] nextJoinId() { return Bytes.toBytes(JOIN_KEY_PREFIX + UUID.randomUUID().toString()); } From 04d62ac6adb97d4e667b4bd6dd48b913edeb72c9 Mon Sep 17 00:00:00 2001 From: maryannxue Date: Tue, 6 Aug 2013 15:44:13 -0400 Subject: [PATCH 007/102] add more implementation to QueryCompiler --- .../phoenix/compile/JoinCompiler.java | 19 +++++++++++- .../phoenix/compile/QueryCompiler.java | 31 +++++++++++++++++-- .../phoenix/compile/StatementContext.java | 13 +++++--- .../phoenix/execute/HashJoinPlan.java | 21 ++++++++----- .../salesforce/phoenix/join/HashJoinInfo.java | 2 +- 5 files changed, 69 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java index 7c4cc65a..481a8180 100644 --- a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java @@ -30,6 +30,8 @@ import java.sql.SQLException; import java.util.List; +import org.apache.hadoop.hbase.util.Pair; + import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.jdbc.PhoenixConnection; import com.salesforce.phoenix.parse.AliasedNode; @@ -81,6 +83,10 @@ public static class JoinTable { private List postJoinFilters; // will be pushed to postFilters in case of star join private TableRef table; private SelectStatement subquery; + + public JoinType getType() { + return type; + } } public interface JoinedColumnResolver extends ColumnResolver { @@ -88,7 +94,7 @@ public interface JoinedColumnResolver extends ColumnResolver { } public static JoinedColumnResolver getResolver(SelectStatement statement, PhoenixConnection connection) throws SQLException { - //TODO + // TODO return null; } @@ -96,4 +102,15 @@ public static StarJoinType getStarJoinType(JoinSpec join) { // TODO return StarJoinType.NONE; } + + // Left: other table expressions; Right: this table expressions. + public static Pair, List> splitEquiJoinConditions(JoinTable joinTable) { + // TODO + return null; + } + + public static SelectStatement newSelectWithoutJoin(SelectStatement statement) { + // TODO + return null; + } } diff --git a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java index 75f8e88e..89eeabb9 100644 --- a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java @@ -33,6 +33,8 @@ import java.util.Map; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Pair; import com.salesforce.phoenix.compile.GroupByCompiler.GroupBy; import com.salesforce.phoenix.compile.JoinCompiler.JoinSpec; @@ -41,12 +43,16 @@ import com.salesforce.phoenix.compile.JoinCompiler.StarJoinType; import com.salesforce.phoenix.compile.OrderByCompiler.OrderBy; import com.salesforce.phoenix.execute.AggregatePlan; +import com.salesforce.phoenix.execute.BasicQueryPlan; +import com.salesforce.phoenix.execute.HashJoinPlan; import com.salesforce.phoenix.execute.ScanPlan; import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.jdbc.PhoenixConnection; import com.salesforce.phoenix.jdbc.PhoenixDatabaseMetaData; import com.salesforce.phoenix.join.HashCacheClient; +import com.salesforce.phoenix.join.HashJoinInfo; import com.salesforce.phoenix.parse.*; +import com.salesforce.phoenix.parse.JoinTableNode.JoinType; import com.salesforce.phoenix.query.QueryConstants; import com.salesforce.phoenix.schema.*; @@ -124,15 +130,34 @@ public QueryPlan compile(SelectStatement statement, List binds) throws S return compile(context, statement, binds); } - protected QueryPlan compile(StatementContext context, SelectStatement statement, List binds, JoinSpec join) { + @SuppressWarnings("unchecked") + protected QueryPlan compile(StatementContext context, SelectStatement statement, List binds, JoinSpec join) throws SQLException { StarJoinType starJoin = JoinCompiler.getStarJoinType(join); if (starJoin == StarJoinType.BASIC || starJoin == StarJoinType.EXTENDED) { - for (JoinTable joinTable : join.getJoinTables()) { + List joinTables = join.getJoinTables(); + int count = joinTables.size(); + ImmutableBytesWritable[] joinIds = new ImmutableBytesWritable[count]; + List[] joinExpressions = (List[]) new List[count]; + List[] hashExpressions = (List[]) new List[count]; + JoinType[] joinTypes = new JoinType[count]; + QueryPlan[] joinPlans = new QueryPlan[count]; + for (int i = 0; i < count; i++) { + joinIds[i] = new ImmutableBytesWritable(HashCacheClient.nextJoinId()); + Pair, List> splittedExpressions = JoinCompiler.splitEquiJoinConditions(joinTables.get(i)); + joinExpressions[i] = splittedExpressions.getFirst(); + hashExpressions[i] = splittedExpressions.getSecond(); + joinTypes[i] = joinTables.get(i).getType(); } + HashJoinInfo joinInfo = new HashJoinInfo(joinIds, joinExpressions, joinTypes); + HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); + BasicQueryPlan plan = compile(context, JoinCompiler.newSelectWithoutJoin(statement), binds); + return new HashJoinPlan(plan, joinIds, hashExpressions, joinPlans); } + + return null; } - protected QueryPlan compile(StatementContext context, SelectStatement statement, List binds) throws SQLException{ + protected BasicQueryPlan compile(StatementContext context, SelectStatement statement, List binds) throws SQLException{ ColumnResolver resolver = context.getResolver(); Map aliasParseNodeMap = ProjectionCompiler.buildAliasParseNodeMap(context, statement.getSelect()); Integer limit = LimitCompiler.getLimit(context, statement.getLimit()); diff --git a/src/main/java/com/salesforce/phoenix/compile/StatementContext.java b/src/main/java/com/salesforce/phoenix/compile/StatementContext.java index e259f368..ffa37c09 100644 --- a/src/main/java/com/salesforce/phoenix/compile/StatementContext.java +++ b/src/main/java/com/salesforce/phoenix/compile/StatementContext.java @@ -27,6 +27,7 @@ ******************************************************************************/ package com.salesforce.phoenix.compile; +import java.io.IOException; import java.sql.SQLException; import java.text.Format; import java.util.List; @@ -73,18 +74,22 @@ public class StatementContext { private long currentTime = QueryConstants.UNSET_TIMESTAMP; private ScanRanges scanRanges = ScanRanges.EVERYTHING; - public StatementContext(PhoenixConnection connection, ColumnResolver resolver, List binds, int bindCount, Scan scan) { + public StatementContext(PhoenixConnection connection, ColumnResolver resolver, List binds, int bindCount, Scan scan) throws SQLException { this(connection, resolver, binds, bindCount, scan, null); } - public StatementContext(PhoenixConnection connection, ColumnResolver resolver, List binds, int bindCount, Scan scan, HintNode hintNode) { + public StatementContext(PhoenixConnection connection, ColumnResolver resolver, List binds, int bindCount, Scan scan, HintNode hintNode) throws SQLException { this(connection, resolver, binds, bindCount, scan, hintNode, null); } - public StatementContext(PhoenixConnection connection, ColumnResolver resolver, List binds, int bindCount, Scan scan, HintNode hintNode, HashCacheClient hashClient) { + public StatementContext(PhoenixConnection connection, ColumnResolver resolver, List binds, int bindCount, Scan scan, HintNode hintNode, HashCacheClient hashClient) throws SQLException { this.connection = connection; this.resolver = resolver; - this.scan = scan; + try { + this.scan = new Scan(scan); // scan cannot be reused for nested query plans + } catch (IOException e) { + throw new SQLException(e); + } this.binds = new BindManager(binds, bindCount); this.aggregates = new AggregationManager(); this.expressions = new ExpressionManager(); diff --git a/src/main/java/com/salesforce/phoenix/execute/HashJoinPlan.java b/src/main/java/com/salesforce/phoenix/execute/HashJoinPlan.java index d68a8e89..63c0a32c 100644 --- a/src/main/java/com/salesforce/phoenix/execute/HashJoinPlan.java +++ b/src/main/java/com/salesforce/phoenix/execute/HashJoinPlan.java @@ -12,7 +12,6 @@ import com.salesforce.phoenix.compile.OrderByCompiler.OrderBy; import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.join.HashCacheClient; -import com.salesforce.phoenix.join.HashJoinInfo; import com.salesforce.phoenix.query.KeyRange; import com.salesforce.phoenix.query.Scanner; import com.salesforce.phoenix.schema.TableRef; @@ -20,8 +19,17 @@ public class HashJoinPlan implements QueryPlan { private BasicQueryPlan plan; - private HashJoinInfo joinInfo; - private List hashPlans; + private ImmutableBytesWritable[] joinIds; + private List[] hashExpressions; + private QueryPlan[] hashPlans; + + public HashJoinPlan(BasicQueryPlan plan, ImmutableBytesWritable[] joinIds, + List[] hashExpressions, QueryPlan[] hashPlans) { + this.plan = plan; + this.joinIds = joinIds; + this.hashExpressions = hashExpressions; + this.hashPlans = hashPlans; + } @Override public Integer getLimit() { @@ -40,15 +48,12 @@ public RowProjector getProjector() { @Override public Scanner getScanner() throws SQLException { - ImmutableBytesWritable[] joinIds = joinInfo.getJoinIds(); - List[] joinExpressions = joinInfo.getJoinExpressions(); - - assert (joinIds.length == joinExpressions.length && joinIds.length == hashPlans.size()); + assert (joinIds.length == hashExpressions.length && joinIds.length == hashPlans.length); HashCacheClient hashClient = plan.getContext().getHashClient(); // TODO replace with Future execution for (int i = 0; i < joinIds.length; i++) { - hashClient.addHashCache(joinIds[i].get(), hashPlans.get(i).getScanner(), joinExpressions[i], plan.getTable().getTableName()); + hashClient.addHashCache(joinIds[i].get(), hashPlans[i].getScanner(), hashExpressions[i], plan.getTable().getTableName()); } return plan.getScanner(); } diff --git a/src/main/java/com/salesforce/phoenix/join/HashJoinInfo.java b/src/main/java/com/salesforce/phoenix/join/HashJoinInfo.java index 80774e11..54136930 100644 --- a/src/main/java/com/salesforce/phoenix/join/HashJoinInfo.java +++ b/src/main/java/com/salesforce/phoenix/join/HashJoinInfo.java @@ -50,7 +50,7 @@ public class HashJoinInfo { private List[] joinExpressions; private JoinType[] joinTypes; - private HashJoinInfo(ImmutableBytesWritable[] joinIds, List[] joinExpressions, JoinType[] joinTypes) { + public HashJoinInfo(ImmutableBytesWritable[] joinIds, List[] joinExpressions, JoinType[] joinTypes) { this.joinIds = joinIds; this.joinExpressions = joinExpressions; this.joinTypes = joinTypes; From 54cd66de6a557c993d484371c8358967677c95a7 Mon Sep 17 00:00:00 2001 From: maryannxue Date: Tue, 6 Aug 2013 16:22:13 -0400 Subject: [PATCH 008/102] fix compilation --- .../phoenix/compile/QueryCompiler.java | 10 +++---- .../phoenix/execute/HashJoinPlan.java | 28 +++++++++++++------ 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java index 24e8b4c6..892d2716 100644 --- a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java @@ -119,12 +119,7 @@ public QueryPlan compile(SelectStatement statement, List binds) throws S List fromNodes = statement.getFrom(); if (fromNodes.size() == 1) { ColumnResolver resolver = FromCompiler.getResolver(statement, connection); - TableRef tableRef = resolver.getTables().get(0); - PTable table = tableRef.getTable(); StatementContext context = new StatementContext(connection, resolver, binds, statement.getBindCount(), scan, statement.getHint()); - if (table.getType() == PTableType.INDEX && table.getIndexState() != PIndexState.ACTIVE) { - return new DegenerateQueryPlan(context, tableRef); - } return compile(context, statement, binds); } @@ -163,6 +158,11 @@ protected QueryPlan compile(StatementContext context, SelectStatement statement, protected BasicQueryPlan compile(StatementContext context, SelectStatement statement, List binds) throws SQLException{ ColumnResolver resolver = context.getResolver(); TableRef tableRef = resolver.getTables().get(0); + PTable table = tableRef.getTable(); + if (table.getType() == PTableType.INDEX && table.getIndexState() != PIndexState.ACTIVE) { + return new DegenerateQueryPlan(context, tableRef); + } + Map aliasParseNodeMap = ProjectionCompiler.buildAliasParseNodeMap(context, statement.getSelect()); Integer limit = LimitCompiler.getLimit(context, statement.getLimit()); diff --git a/src/main/java/com/salesforce/phoenix/execute/HashJoinPlan.java b/src/main/java/com/salesforce/phoenix/execute/HashJoinPlan.java index 63c0a32c..cb22abf0 100644 --- a/src/main/java/com/salesforce/phoenix/execute/HashJoinPlan.java +++ b/src/main/java/com/salesforce/phoenix/execute/HashJoinPlan.java @@ -9,6 +9,8 @@ import com.salesforce.phoenix.compile.ExplainPlan; import com.salesforce.phoenix.compile.QueryPlan; import com.salesforce.phoenix.compile.RowProjector; +import com.salesforce.phoenix.compile.StatementContext; +import com.salesforce.phoenix.compile.GroupByCompiler.GroupBy; import com.salesforce.phoenix.compile.OrderByCompiler.OrderBy; import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.join.HashCacheClient; @@ -53,7 +55,7 @@ public Scanner getScanner() throws SQLException { HashCacheClient hashClient = plan.getContext().getHashClient(); // TODO replace with Future execution for (int i = 0; i < joinIds.length; i++) { - hashClient.addHashCache(joinIds[i].get(), hashPlans[i].getScanner(), hashExpressions[i], plan.getTable().getTableName()); + hashClient.addHashCache(joinIds[i].get(), hashPlans[i].getScanner(), hashExpressions[i], plan.getTableRef()); } return plan.getScanner(); } @@ -64,23 +66,31 @@ public List getSplits() { } @Override - public TableRef getTable() { - return plan.getTable(); + public ExplainPlan getExplainPlan() throws SQLException { + return plan.getExplainPlan(); } @Override - public boolean isAggregate() { - return plan.isAggregate(); + public ParameterMetaData getParameterMetaData() { + return plan.getParameterMetaData(); } @Override - public ExplainPlan getExplainPlan() throws SQLException { - return plan.getExplainPlan(); + public StatementContext getContext() { + // TODO Auto-generated method stub + return null; } @Override - public ParameterMetaData getParameterMetaData() { - return plan.getParameterMetaData(); + public GroupBy getGroupBy() { + // TODO Auto-generated method stub + return null; + } + + @Override + public TableRef getTableRef() { + // TODO Auto-generated method stub + return null; } } From 05bc6f21eee30d1b6157e13631633e91c6089979 Mon Sep 17 00:00:00 2001 From: maryannxue Date: Thu, 8 Aug 2013 18:08:52 -0400 Subject: [PATCH 009/102] Add more querycompiler impl --- .../phoenix/compile/JoinCompiler.java | 45 ++++++++++++++++--- .../phoenix/compile/QueryCompiler.java | 41 ++++++++++++++--- 2 files changed, 72 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java index 481a8180..bac452c7 100644 --- a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java @@ -55,24 +55,35 @@ public static class JoinSpec { private List postFilters; private List joinTables; private boolean isPostAggregate; + private ColumnResolver resolver; private JoinSpec(TableRef table, List select, List filters, - List postFilters, List joinTables, boolean isPostAggregate) { + List postFilters, List joinTables, boolean isPostAggregate, + ColumnResolver resolver) { this.table = table; this.select = select; this.filters = filters; this.postFilters = postFilters; this.joinTables = joinTables; this.isPostAggregate = isPostAggregate; + this.resolver = resolver; } public List getJoinTables() { return joinTables; } + + public boolean isPostAggregate() { + return isPostAggregate; + } + + public ColumnResolver getColumnResolver() { + return resolver; + } } public static JoinSpec getSubJoinSpec(JoinSpec join) { - return new JoinSpec(join.table, join.select, join.filters, join.postFilters, join.joinTables.subList(0, join.joinTables.size() - 2), join.isPostAggregate); + return new JoinSpec(join.table, join.select, join.filters, join.postFilters, join.joinTables.subList(0, join.joinTables.size() - 2), join.isPostAggregate, join.resolver); } public static class JoinTable { @@ -87,13 +98,14 @@ public static class JoinTable { public JoinType getType() { return type; } + + public SelectStatement getAsSubquery() { + // TODO + return subquery; + } } - public interface JoinedColumnResolver extends ColumnResolver { - public JoinSpec getJoinTables(); - } - - public static JoinedColumnResolver getResolver(SelectStatement statement, PhoenixConnection connection) throws SQLException { + public static JoinSpec getJoinSpec(SelectStatement statement, PhoenixConnection connection) throws SQLException { // TODO return null; } @@ -113,4 +125,23 @@ public static SelectStatement newSelectWithoutJoin(SelectStatement statement) { // TODO return null; } + + // Get the last join table select statement with fixed-up select and where nodes. + // Currently does NOT support last join table as a subquery. + public static SelectStatement newSelectForLastJoin(SelectStatement statement, JoinSpec join) { + // TODO + return null; + } + + // Get subquery with fixed select and where nodes + public static SelectStatement getSubQuery(SelectStatement statement) { + // TODO + return null; + } + + // Get subquery with complete select and where nodes + public static SelectStatement getSubQueryForFinalPlan(SelectStatement statement) { + // TODO + return null; + } } diff --git a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java index 892d2716..09310723 100644 --- a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java @@ -39,7 +39,6 @@ import com.salesforce.phoenix.compile.GroupByCompiler.GroupBy; import com.salesforce.phoenix.compile.JoinCompiler.JoinSpec; import com.salesforce.phoenix.compile.JoinCompiler.JoinTable; -import com.salesforce.phoenix.compile.JoinCompiler.JoinedColumnResolver; import com.salesforce.phoenix.compile.JoinCompiler.StarJoinType; import com.salesforce.phoenix.compile.OrderByCompiler.OrderBy; import com.salesforce.phoenix.execute.*; @@ -123,16 +122,20 @@ public QueryPlan compile(SelectStatement statement, List binds) throws S return compile(context, statement, binds); } - JoinedColumnResolver resolver = JoinCompiler.getResolver(statement, connection); - StatementContext context = new StatementContext(connection, resolver, binds, statement.getBindCount(), scan, statement.getHint(), new HashCacheClient(connection.getQueryServices(), connection.getTenantId())); - return compile(context, statement, binds); + JoinSpec join = JoinCompiler.getJoinSpec(statement, connection); + StatementContext context = new StatementContext(connection, join.getColumnResolver(), binds, statement.getBindCount(), scan, statement.getHint(), new HashCacheClient(connection.getQueryServices(), connection.getTenantId())); + return compile(context, statement, binds, join); } @SuppressWarnings("unchecked") protected QueryPlan compile(StatementContext context, SelectStatement statement, List binds, JoinSpec join) throws SQLException { + List joinTables = join.getJoinTables(); + if (joinTables.isEmpty()) { + return compile(context, statement, binds); + } + StarJoinType starJoin = JoinCompiler.getStarJoinType(join); if (starJoin == StarJoinType.BASIC || starJoin == StarJoinType.EXTENDED) { - List joinTables = join.getJoinTables(); int count = joinTables.size(); ImmutableBytesWritable[] joinIds = new ImmutableBytesWritable[count]; List[] joinExpressions = (List[]) new List[count]; @@ -140,11 +143,13 @@ protected QueryPlan compile(StatementContext context, SelectStatement statement, JoinType[] joinTypes = new JoinType[count]; QueryPlan[] joinPlans = new QueryPlan[count]; for (int i = 0; i < count; i++) { + JoinTable joinTable = joinTables.get(i); joinIds[i] = new ImmutableBytesWritable(HashCacheClient.nextJoinId()); - Pair, List> splittedExpressions = JoinCompiler.splitEquiJoinConditions(joinTables.get(i)); + Pair, List> splittedExpressions = JoinCompiler.splitEquiJoinConditions(joinTable); joinExpressions[i] = splittedExpressions.getFirst(); hashExpressions[i] = splittedExpressions.getSecond(); - joinTypes[i] = joinTables.get(i).getType(); + joinTypes[i] = joinTable.getType(); + joinPlans[i] = compile(joinTable.getAsSubquery(), binds); } HashJoinInfo joinInfo = new HashJoinInfo(joinIds, joinExpressions, joinTypes); HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); @@ -152,6 +157,28 @@ protected QueryPlan compile(StatementContext context, SelectStatement statement, return new HashJoinPlan(plan, joinIds, hashExpressions, joinPlans); } + JoinTable lastJoinTable = joinTables.get(joinTables.size() - 1); + JoinType type = lastJoinTable.getType(); + if (type == JoinType.Right) { + if (join.isPostAggregate()) { + throw new UnsupportedOperationException("Does not support aggregation functions on right join."); + } + SelectStatement lhs = JoinCompiler.getSubQuery(statement); + SelectStatement rhs = JoinCompiler.newSelectForLastJoin(statement, join); + JoinSpec lhsJoin = JoinCompiler.getSubJoinSpec(join); + StatementContext lhsCtx = new StatementContext(connection, join.getColumnResolver(), binds, statement.getBindCount(), scan, statement.getHint(), new HashCacheClient(connection.getQueryServices(), connection.getTenantId())); + QueryPlan lhsPlan = compile(lhsCtx, lhs, binds, lhsJoin); + byte[] joinId = HashCacheClient.nextJoinId(); + ImmutableBytesWritable[] joinIds = new ImmutableBytesWritable[] {new ImmutableBytesWritable(joinId)}; + Pair, List> splittedExpressions = JoinCompiler.splitEquiJoinConditions(lastJoinTable); + List joinExpressions = splittedExpressions.getSecond(); + List hashExpressions = splittedExpressions.getFirst(); + HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {joinExpressions}, new JoinType[] {JoinType.Left}); + HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); + BasicQueryPlan rhsPlan = compile(context, rhs, binds); + return new HashJoinPlan(rhsPlan, joinIds, new List[] {hashExpressions}, new QueryPlan[] {lhsPlan}); + } + return null; } From 806d1199bcf2f4aa5b024cd14c843c47256a76a1 Mon Sep 17 00:00:00 2001 From: maryannxue Date: Fri, 9 Aug 2013 16:01:46 -0400 Subject: [PATCH 010/102] Complete QueryCompiler for joins --- .../phoenix/compile/JoinCompiler.java | 60 +++++++++++++++++-- .../phoenix/compile/QueryCompiler.java | 22 +++++-- 2 files changed, 74 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java index bac452c7..54070ed7 100644 --- a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java @@ -49,7 +49,7 @@ public enum StarJoinType { } public static class JoinSpec { - private TableRef table; + private TableRef mainTable; private List select; private List filters; private List postFilters; @@ -60,7 +60,7 @@ public static class JoinSpec { private JoinSpec(TableRef table, List select, List filters, List postFilters, List joinTables, boolean isPostAggregate, ColumnResolver resolver) { - this.table = table; + this.mainTable = table; this.select = select; this.filters = filters; this.postFilters = postFilters; @@ -68,7 +68,23 @@ private JoinSpec(TableRef table, List select, List filt this.isPostAggregate = isPostAggregate; this.resolver = resolver; } - + + public TableRef getMainTable() { + return mainTable; + } + + public List getSelect() { + return select; + } + + public List getFilters() { + return filters; + } + + public List getPostFilters() { + return postFilters; + } + public List getJoinTables() { return joinTables; } @@ -83,7 +99,7 @@ public ColumnResolver getColumnResolver() { } public static JoinSpec getSubJoinSpec(JoinSpec join) { - return new JoinSpec(join.table, join.select, join.filters, join.postFilters, join.joinTables.subList(0, join.joinTables.size() - 2), join.isPostAggregate, join.resolver); + return new JoinSpec(join.mainTable, join.select, join.filters, join.postFilters, join.joinTables.subList(0, join.joinTables.size() - 2), join.isPostAggregate, join.resolver); } public static class JoinTable { @@ -95,10 +111,45 @@ public static class JoinTable { private TableRef table; private SelectStatement subquery; + private JoinTable(JoinType type, List conditions, List select, + List filters, List postJoinFilters, TableRef table, SelectStatement subquery) { + this.type = type; + this.conditions = conditions; + this.select = select; + this.filters = filters; + this.postJoinFilters = postJoinFilters; + this.table = table; + this.subquery = subquery; + } + public JoinType getType() { return type; } + public List getJoinConditions() { + return conditions; + } + + public List getSelect() { + return select; + } + + public List getFilters() { + return filters; + } + + public List getPostJoinFilters() { + return postJoinFilters; + } + + public TableRef getTable() { + return table; + } + + public SelectStatement getSubquery() { + return subquery; + } + public SelectStatement getAsSubquery() { // TODO return subquery; @@ -140,6 +191,7 @@ public static SelectStatement getSubQuery(SelectStatement statement) { } // Get subquery with complete select and where nodes + // Throws exception if the subquery contains joins. public static SelectStatement getSubQueryForFinalPlan(SelectStatement statement) { // TODO return null; diff --git a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java index 09310723..616c01e4 100644 --- a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java @@ -130,9 +130,8 @@ public QueryPlan compile(SelectStatement statement, List binds) throws S @SuppressWarnings("unchecked") protected QueryPlan compile(StatementContext context, SelectStatement statement, List binds, JoinSpec join) throws SQLException { List joinTables = join.getJoinTables(); - if (joinTables.isEmpty()) { + if (joinTables.isEmpty()) return compile(context, statement, binds); - } StarJoinType starJoin = JoinCompiler.getStarJoinType(join); if (starJoin == StarJoinType.BASIC || starJoin == StarJoinType.EXTENDED) { @@ -159,7 +158,11 @@ protected QueryPlan compile(StatementContext context, SelectStatement statement, JoinTable lastJoinTable = joinTables.get(joinTables.size() - 1); JoinType type = lastJoinTable.getType(); - if (type == JoinType.Right) { + if (type == JoinType.Full) + throw new UnsupportedOperationException("Does not support full join."); + + if (type == JoinType.Right + || (type == JoinType.Inner && joinTables.size() > 1)) { if (join.isPostAggregate()) { throw new UnsupportedOperationException("Does not support aggregation functions on right join."); } @@ -179,7 +182,18 @@ protected QueryPlan compile(StatementContext context, SelectStatement statement, return new HashJoinPlan(rhsPlan, joinIds, new List[] {hashExpressions}, new QueryPlan[] {lhsPlan}); } - return null; + SelectStatement lhs = JoinCompiler.getSubQueryForFinalPlan(statement); + SelectStatement rhs = lastJoinTable.getAsSubquery(); + QueryPlan rhsPlan = compile(rhs, binds); + byte[] joinId = HashCacheClient.nextJoinId(); + ImmutableBytesWritable[] joinIds = new ImmutableBytesWritable[] {new ImmutableBytesWritable(joinId)}; + Pair, List> splittedExpressions = JoinCompiler.splitEquiJoinConditions(lastJoinTable); + List joinExpressions = splittedExpressions.getFirst(); + List hashExpressions = splittedExpressions.getSecond(); + HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {joinExpressions}, new JoinType[] {JoinType.Left}); + HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); + BasicQueryPlan lhsPlan = compile(context, lhs, binds); + return new HashJoinPlan(lhsPlan, joinIds, new List[] {hashExpressions}, new QueryPlan[] {rhsPlan}); } protected BasicQueryPlan compile(StatementContext context, SelectStatement statement, List binds) throws SQLException{ From fb9dc7186e4ce6f6619b27f78344d53cfb314be5 Mon Sep 17 00:00:00 2001 From: maryannxue Date: Mon, 12 Aug 2013 21:35:41 -0400 Subject: [PATCH 011/102] Fix UT regressions --- .../phoenix/compile/QueryCompiler.java | 30 ++++++++++++------- .../phoenix/compile/StatementContext.java | 13 +++----- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java index 616c01e4..e10c9467 100644 --- a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java @@ -27,6 +27,7 @@ ******************************************************************************/ package com.salesforce.phoenix.compile; +import java.io.IOException; import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; import java.util.List; @@ -114,24 +115,31 @@ public QueryPlan compile(SelectStatement statement, List binds) throws S assert(binds.size() == statement.getBindCount()); + Scan s; + try { + s = new Scan(scan); // scan cannot be reused for nested query plans + } catch (IOException e) { + throw new SQLException(e); + } + statement = RHSLiteralStatementRewriter.normalize(statement); List fromNodes = statement.getFrom(); if (fromNodes.size() == 1) { ColumnResolver resolver = FromCompiler.getResolver(statement, connection); - StatementContext context = new StatementContext(connection, resolver, binds, statement.getBindCount(), scan, statement.getHint()); - return compile(context, statement, binds); + StatementContext context = new StatementContext(connection, resolver, binds, statement.getBindCount(), s, statement.getHint()); + return compileSingleQuery(context, statement, binds); } JoinSpec join = JoinCompiler.getJoinSpec(statement, connection); - StatementContext context = new StatementContext(connection, join.getColumnResolver(), binds, statement.getBindCount(), scan, statement.getHint(), new HashCacheClient(connection.getQueryServices(), connection.getTenantId())); - return compile(context, statement, binds, join); + StatementContext context = new StatementContext(connection, join.getColumnResolver(), binds, statement.getBindCount(), s, statement.getHint(), new HashCacheClient(connection.getQueryServices(), connection.getTenantId())); + return compileJoinQuery(context, statement, binds, join); } @SuppressWarnings("unchecked") - protected QueryPlan compile(StatementContext context, SelectStatement statement, List binds, JoinSpec join) throws SQLException { + protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement statement, List binds, JoinSpec join) throws SQLException { List joinTables = join.getJoinTables(); if (joinTables.isEmpty()) - return compile(context, statement, binds); + return compileSingleQuery(context, statement, binds); StarJoinType starJoin = JoinCompiler.getStarJoinType(join); if (starJoin == StarJoinType.BASIC || starJoin == StarJoinType.EXTENDED) { @@ -152,7 +160,7 @@ protected QueryPlan compile(StatementContext context, SelectStatement statement, } HashJoinInfo joinInfo = new HashJoinInfo(joinIds, joinExpressions, joinTypes); HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); - BasicQueryPlan plan = compile(context, JoinCompiler.newSelectWithoutJoin(statement), binds); + BasicQueryPlan plan = compileSingleQuery(context, JoinCompiler.newSelectWithoutJoin(statement), binds); return new HashJoinPlan(plan, joinIds, hashExpressions, joinPlans); } @@ -170,7 +178,7 @@ protected QueryPlan compile(StatementContext context, SelectStatement statement, SelectStatement rhs = JoinCompiler.newSelectForLastJoin(statement, join); JoinSpec lhsJoin = JoinCompiler.getSubJoinSpec(join); StatementContext lhsCtx = new StatementContext(connection, join.getColumnResolver(), binds, statement.getBindCount(), scan, statement.getHint(), new HashCacheClient(connection.getQueryServices(), connection.getTenantId())); - QueryPlan lhsPlan = compile(lhsCtx, lhs, binds, lhsJoin); + QueryPlan lhsPlan = compileJoinQuery(lhsCtx, lhs, binds, lhsJoin); byte[] joinId = HashCacheClient.nextJoinId(); ImmutableBytesWritable[] joinIds = new ImmutableBytesWritable[] {new ImmutableBytesWritable(joinId)}; Pair, List> splittedExpressions = JoinCompiler.splitEquiJoinConditions(lastJoinTable); @@ -178,7 +186,7 @@ protected QueryPlan compile(StatementContext context, SelectStatement statement, List hashExpressions = splittedExpressions.getFirst(); HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {joinExpressions}, new JoinType[] {JoinType.Left}); HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); - BasicQueryPlan rhsPlan = compile(context, rhs, binds); + BasicQueryPlan rhsPlan = compileSingleQuery(context, rhs, binds); return new HashJoinPlan(rhsPlan, joinIds, new List[] {hashExpressions}, new QueryPlan[] {lhsPlan}); } @@ -192,11 +200,11 @@ protected QueryPlan compile(StatementContext context, SelectStatement statement, List hashExpressions = splittedExpressions.getSecond(); HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {joinExpressions}, new JoinType[] {JoinType.Left}); HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); - BasicQueryPlan lhsPlan = compile(context, lhs, binds); + BasicQueryPlan lhsPlan = compileSingleQuery(context, lhs, binds); return new HashJoinPlan(lhsPlan, joinIds, new List[] {hashExpressions}, new QueryPlan[] {rhsPlan}); } - protected BasicQueryPlan compile(StatementContext context, SelectStatement statement, List binds) throws SQLException{ + protected BasicQueryPlan compileSingleQuery(StatementContext context, SelectStatement statement, List binds) throws SQLException{ ColumnResolver resolver = context.getResolver(); TableRef tableRef = resolver.getTables().get(0); PTable table = tableRef.getTable(); diff --git a/src/main/java/com/salesforce/phoenix/compile/StatementContext.java b/src/main/java/com/salesforce/phoenix/compile/StatementContext.java index b05fc9cc..8a607886 100644 --- a/src/main/java/com/salesforce/phoenix/compile/StatementContext.java +++ b/src/main/java/com/salesforce/phoenix/compile/StatementContext.java @@ -27,7 +27,6 @@ ******************************************************************************/ package com.salesforce.phoenix.compile; -import java.io.IOException; import java.sql.SQLException; import java.text.Format; import java.util.List; @@ -74,22 +73,18 @@ public class StatementContext { private long currentTime = QueryConstants.UNSET_TIMESTAMP; private ScanRanges scanRanges = ScanRanges.EVERYTHING; - public StatementContext(PhoenixConnection connection, ColumnResolver resolver, List binds, int bindCount, Scan scan) throws SQLException { + public StatementContext(PhoenixConnection connection, ColumnResolver resolver, List binds, int bindCount, Scan scan) { this(connection, resolver, binds, bindCount, scan, null); } - public StatementContext(PhoenixConnection connection, ColumnResolver resolver, List binds, int bindCount, Scan scan, HintNode hintNode) throws SQLException { + public StatementContext(PhoenixConnection connection, ColumnResolver resolver, List binds, int bindCount, Scan scan, HintNode hintNode) { this(connection, resolver, binds, bindCount, scan, hintNode, null); } - public StatementContext(PhoenixConnection connection, ColumnResolver resolver, List binds, int bindCount, Scan scan, HintNode hintNode, HashCacheClient hashClient) throws SQLException { + public StatementContext(PhoenixConnection connection, ColumnResolver resolver, List binds, int bindCount, Scan scan, HintNode hintNode, HashCacheClient hashClient) { this.connection = connection; this.resolver = resolver; - try { - this.scan = new Scan(scan); // scan cannot be reused for nested query plans - } catch (IOException e) { - throw new SQLException(e); - } + this.scan = scan; this.binds = new BindManager(binds, bindCount); this.aggregates = new AggregationManager(); this.expressions = new ExpressionManager(); From 23d77dcb6139956fd4d9923d9b09b5d560fee26e Mon Sep 17 00:00:00 2001 From: maryannxue Date: Tue, 13 Aug 2013 16:41:17 -0400 Subject: [PATCH 012/102] fix unit test --- .../phoenix/compile/QueryCompiler.java | 43 ++++++++++++------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java index e10c9467..955da072 100644 --- a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java @@ -74,22 +74,23 @@ public class QueryCompiler { private static final String LOAD_COLUMN_FAMILIES_ON_DEMAND_ATTR = "_ondemand_"; private final PhoenixConnection connection; private final Scan scan; + private final Scan scanCopy; private final int maxRows; private final PColumn[] targetColumns; - public QueryCompiler(PhoenixConnection connection, int maxRows) { + public QueryCompiler(PhoenixConnection connection, int maxRows) throws SQLException { this(connection, maxRows, new Scan()); } - public QueryCompiler(PhoenixConnection connection, int maxRows, Scan scan) { + public QueryCompiler(PhoenixConnection connection, int maxRows, Scan scan) throws SQLException { this(connection, maxRows, scan, null); } - public QueryCompiler(PhoenixConnection connection, int maxRows, PColumn[] targetDatums) { + public QueryCompiler(PhoenixConnection connection, int maxRows, PColumn[] targetDatums) throws SQLException { this(connection, maxRows, new Scan(), targetDatums); } - public QueryCompiler(PhoenixConnection connection, int maxRows, Scan scan, PColumn[] targetDatums) { + public QueryCompiler(PhoenixConnection connection, int maxRows, Scan scan, PColumn[] targetDatums) throws SQLException { this.connection = connection; this.maxRows = maxRows; this.scan = scan; @@ -97,6 +98,11 @@ public QueryCompiler(PhoenixConnection connection, int maxRows, Scan scan, PColu if (connection.getQueryServices().getLowestClusterHBaseVersion() >= PhoenixDatabaseMetaData.ESSENTIAL_FAMILY_VERSION_THRESHOLD) { this.scan.setAttribute(LOAD_COLUMN_FAMILIES_ON_DEMAND_ATTR, QueryConstants.TRUE); } + try { + this.scanCopy = new Scan(scan); + } catch (IOException e) { + throw new SQLException(e); + } } /** @@ -112,26 +118,22 @@ public QueryCompiler(PhoenixConnection connection, int maxRows, Scan scan, PColu * @throws AmbiguousColumnException if an unaliased column name is ambiguous across multiple tables */ public QueryPlan compile(SelectStatement statement, List binds) throws SQLException{ - + return compile(statement, binds, scan); + } + + protected QueryPlan compile(SelectStatement statement, List binds, Scan scan) throws SQLException{ assert(binds.size() == statement.getBindCount()); - Scan s; - try { - s = new Scan(scan); // scan cannot be reused for nested query plans - } catch (IOException e) { - throw new SQLException(e); - } - statement = RHSLiteralStatementRewriter.normalize(statement); List fromNodes = statement.getFrom(); if (fromNodes.size() == 1) { ColumnResolver resolver = FromCompiler.getResolver(statement, connection); - StatementContext context = new StatementContext(connection, resolver, binds, statement.getBindCount(), s, statement.getHint()); + StatementContext context = new StatementContext(connection, resolver, binds, statement.getBindCount(), scan, statement.getHint()); return compileSingleQuery(context, statement, binds); } JoinSpec join = JoinCompiler.getJoinSpec(statement, connection); - StatementContext context = new StatementContext(connection, join.getColumnResolver(), binds, statement.getBindCount(), s, statement.getHint(), new HashCacheClient(connection.getQueryServices(), connection.getTenantId())); + StatementContext context = new StatementContext(connection, join.getColumnResolver(), binds, statement.getBindCount(), scan, statement.getHint(), new HashCacheClient(connection.getQueryServices(), connection.getTenantId())); return compileJoinQuery(context, statement, binds, join); } @@ -156,7 +158,11 @@ protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement s joinExpressions[i] = splittedExpressions.getFirst(); hashExpressions[i] = splittedExpressions.getSecond(); joinTypes[i] = joinTable.getType(); - joinPlans[i] = compile(joinTable.getAsSubquery(), binds); + try { + joinPlans[i] = compile(joinTable.getAsSubquery(), binds, new Scan(scanCopy)); + } catch (IOException e) { + throw new SQLException(e); + } } HashJoinInfo joinInfo = new HashJoinInfo(joinIds, joinExpressions, joinTypes); HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); @@ -192,7 +198,12 @@ protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement s SelectStatement lhs = JoinCompiler.getSubQueryForFinalPlan(statement); SelectStatement rhs = lastJoinTable.getAsSubquery(); - QueryPlan rhsPlan = compile(rhs, binds); + QueryPlan rhsPlan; + try { + rhsPlan = compile(rhs, binds, new Scan(scanCopy)); + } catch (IOException e) { + throw new SQLException(e); + } byte[] joinId = HashCacheClient.nextJoinId(); ImmutableBytesWritable[] joinIds = new ImmutableBytesWritable[] {new ImmutableBytesWritable(joinId)}; Pair, List> splittedExpressions = JoinCompiler.splitEquiJoinConditions(lastJoinTable); From 0b98d14621706c6a48e8bf3ea7f7d487ea107206 Mon Sep 17 00:00:00 2001 From: maryannxue Date: Thu, 15 Aug 2013 15:04:17 -0400 Subject: [PATCH 013/102] add post join filter in runtime --- .../phoenix/compile/JoinCompiler.java | 85 ++++++++++++------- .../phoenix/compile/QueryCompiler.java | 20 ++--- .../coprocessor/HashJoinRegionScanner.java | 23 +++++ .../salesforce/phoenix/join/HashJoinInfo.java | 22 ++++- 4 files changed, 109 insertions(+), 41 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java index 54070ed7..d7d6cf3d 100644 --- a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java @@ -28,6 +28,7 @@ package com.salesforce.phoenix.compile; import java.sql.SQLException; +import java.util.ArrayList; import java.util.List; import org.apache.hadoop.hbase.util.Pair; @@ -35,7 +36,10 @@ import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.jdbc.PhoenixConnection; import com.salesforce.phoenix.parse.AliasedNode; +import com.salesforce.phoenix.parse.ParseNode; +import com.salesforce.phoenix.parse.ParseNodeFactory; import com.salesforce.phoenix.parse.SelectStatement; +import com.salesforce.phoenix.parse.TableNode; import com.salesforce.phoenix.parse.JoinTableNode.JoinType; import com.salesforce.phoenix.schema.TableRef; @@ -51,21 +55,21 @@ public enum StarJoinType { public static class JoinSpec { private TableRef mainTable; private List select; - private List filters; + private ParseNode preFilters; private List postFilters; private List joinTables; - private boolean isPostAggregate; + private boolean isPostAggregateOrDistinct; private ColumnResolver resolver; - private JoinSpec(TableRef table, List select, List filters, + private JoinSpec(TableRef table, List select, ParseNode preFilters, List postFilters, List joinTables, boolean isPostAggregate, ColumnResolver resolver) { this.mainTable = table; this.select = select; - this.filters = filters; + this.preFilters = preFilters; this.postFilters = postFilters; this.joinTables = joinTables; - this.isPostAggregate = isPostAggregate; + this.isPostAggregateOrDistinct = isPostAggregate; this.resolver = resolver; } @@ -77,8 +81,8 @@ public List getSelect() { return select; } - public List getFilters() { - return filters; + public ParseNode getPreFilters() { + return preFilters; } public List getPostFilters() { @@ -89,8 +93,8 @@ public List getJoinTables() { return joinTables; } - public boolean isPostAggregate() { - return isPostAggregate; + public boolean isPostAggregateOrDistinct() { + return isPostAggregateOrDistinct; } public ColumnResolver getColumnResolver() { @@ -99,24 +103,26 @@ public ColumnResolver getColumnResolver() { } public static JoinSpec getSubJoinSpec(JoinSpec join) { - return new JoinSpec(join.mainTable, join.select, join.filters, join.postFilters, join.joinTables.subList(0, join.joinTables.size() - 2), join.isPostAggregate, join.resolver); + return new JoinSpec(join.mainTable, join.select, join.preFilters, join.postFilters, join.joinTables.subList(0, join.joinTables.size() - 2), join.isPostAggregateOrDistinct, join.resolver); } public static class JoinTable { private JoinType type; private List conditions; + private TableNode tableNode; // original table node + private TableRef table; private List select; - private List filters; + private ParseNode preFilters; private List postJoinFilters; // will be pushed to postFilters in case of star join - private TableRef table; private SelectStatement subquery; - private JoinTable(JoinType type, List conditions, List select, - List filters, List postJoinFilters, TableRef table, SelectStatement subquery) { + private JoinTable(JoinType type, List conditions, TableNode tableNode, List select, + ParseNode preFilters, List postJoinFilters, TableRef table, SelectStatement subquery) { this.type = type; this.conditions = conditions; + this.tableNode = tableNode; this.select = select; - this.filters = filters; + this.preFilters = preFilters; this.postJoinFilters = postJoinFilters; this.table = table; this.subquery = subquery; @@ -130,22 +136,26 @@ public List getJoinConditions() { return conditions; } + public TableNode getTableNode() { + return tableNode; + } + + public TableRef getTable() { + return table; + } + public List getSelect() { return select; } - public List getFilters() { - return filters; + public ParseNode getPreFilters() { + return preFilters; } public List getPostJoinFilters() { return postJoinFilters; } - public TableRef getTable() { - return table; - } - public SelectStatement getSubquery() { return subquery; } @@ -156,6 +166,9 @@ public SelectStatement getAsSubquery() { } } + // for creation of new statements + private static ParseNodeFactory factory = new ParseNodeFactory(); + public static JoinSpec getJoinSpec(SelectStatement statement, PhoenixConnection connection) throws SQLException { // TODO return null; @@ -172,27 +185,41 @@ public static Pair, List> splitEquiJoinConditions(J return null; } - public static SelectStatement newSelectWithoutJoin(SelectStatement statement) { - // TODO - return null; + public static SelectStatement getSubqueryWithoutJoin(SelectStatement statement, JoinSpec join) { + return factory.select(statement.getFrom().subList(0, 1), statement.getHint(), statement.isDistinct(), statement.getSelect(), join.getPreFilters(), statement.getGroupBy(), statement.getHaving(), statement.getOrderBy(), statement.getLimit(), statement.getBindCount()); } // Get the last join table select statement with fixed-up select and where nodes. // Currently does NOT support last join table as a subquery. - public static SelectStatement newSelectForLastJoin(SelectStatement statement, JoinSpec join) { - // TODO - return null; + public static SelectStatement getSubqueryForLastJoinTable(SelectStatement statement, JoinSpec join) { + List joinTables = join.getJoinTables(); + int count = joinTables.size(); + assert (count > 0); + JoinTable lastJoinTable = joinTables.get(count - 1); + if (lastJoinTable.getSubquery() != null) { + throw new UnsupportedOperationException("Right join table cannot be a subquery."); + } + List from = new ArrayList(1); + from.add(lastJoinTable.getTableNode()); + + // TODO distinguish last join in original query or last join in subquery + return factory.select(from, statement.getHint(), statement.isDistinct(), lastJoinTable.getSelect(), lastJoinTable.getPreFilters(), statement.getGroupBy(), statement.getHaving(), statement.getOrderBy(), statement.getLimit(), statement.getBindCount()); } // Get subquery with fixed select and where nodes - public static SelectStatement getSubQuery(SelectStatement statement) { + public static SelectStatement getSubQueryWithoutLastJoin(SelectStatement statement) { // TODO return null; } // Get subquery with complete select and where nodes // Throws exception if the subquery contains joins. - public static SelectStatement getSubQueryForFinalPlan(SelectStatement statement) { + public static SelectStatement getSubQueryWithoutLastJoinAsFinalPlan(SelectStatement statement) { + // TODO + return null; + } + + public static Expression getPostJoinFilterExpression(JoinSpec join, JoinTable joinTable) { // TODO return null; } diff --git a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java index 955da072..3e1e0b52 100644 --- a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java @@ -164,9 +164,10 @@ protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement s throw new SQLException(e); } } - HashJoinInfo joinInfo = new HashJoinInfo(joinIds, joinExpressions, joinTypes); + Expression postJoinFilterExpression = JoinCompiler.getPostJoinFilterExpression(join, null); + HashJoinInfo joinInfo = new HashJoinInfo(joinIds, joinExpressions, joinTypes, postJoinFilterExpression); HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); - BasicQueryPlan plan = compileSingleQuery(context, JoinCompiler.newSelectWithoutJoin(statement), binds); + BasicQueryPlan plan = compileSingleQuery(context, JoinCompiler.getSubqueryWithoutJoin(statement, join), binds); return new HashJoinPlan(plan, joinIds, hashExpressions, joinPlans); } @@ -177,11 +178,8 @@ protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement s if (type == JoinType.Right || (type == JoinType.Inner && joinTables.size() > 1)) { - if (join.isPostAggregate()) { - throw new UnsupportedOperationException("Does not support aggregation functions on right join."); - } - SelectStatement lhs = JoinCompiler.getSubQuery(statement); - SelectStatement rhs = JoinCompiler.newSelectForLastJoin(statement, join); + SelectStatement lhs = JoinCompiler.getSubQueryWithoutLastJoin(statement); + SelectStatement rhs = JoinCompiler.getSubqueryForLastJoinTable(statement, join); JoinSpec lhsJoin = JoinCompiler.getSubJoinSpec(join); StatementContext lhsCtx = new StatementContext(connection, join.getColumnResolver(), binds, statement.getBindCount(), scan, statement.getHint(), new HashCacheClient(connection.getQueryServices(), connection.getTenantId())); QueryPlan lhsPlan = compileJoinQuery(lhsCtx, lhs, binds, lhsJoin); @@ -190,13 +188,14 @@ protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement s Pair, List> splittedExpressions = JoinCompiler.splitEquiJoinConditions(lastJoinTable); List joinExpressions = splittedExpressions.getSecond(); List hashExpressions = splittedExpressions.getFirst(); - HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {joinExpressions}, new JoinType[] {JoinType.Left}); + Expression postJoinFilterExpression = JoinCompiler.getPostJoinFilterExpression(join, lastJoinTable); + HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {joinExpressions}, new JoinType[] {JoinType.Left}, postJoinFilterExpression); HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); BasicQueryPlan rhsPlan = compileSingleQuery(context, rhs, binds); return new HashJoinPlan(rhsPlan, joinIds, new List[] {hashExpressions}, new QueryPlan[] {lhsPlan}); } - SelectStatement lhs = JoinCompiler.getSubQueryForFinalPlan(statement); + SelectStatement lhs = JoinCompiler.getSubQueryWithoutLastJoinAsFinalPlan(statement); SelectStatement rhs = lastJoinTable.getAsSubquery(); QueryPlan rhsPlan; try { @@ -209,7 +208,8 @@ protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement s Pair, List> splittedExpressions = JoinCompiler.splitEquiJoinConditions(lastJoinTable); List joinExpressions = splittedExpressions.getFirst(); List hashExpressions = splittedExpressions.getSecond(); - HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {joinExpressions}, new JoinType[] {JoinType.Left}); + Expression postJoinFilterExpression = JoinCompiler.getPostJoinFilterExpression(join, null); + HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {joinExpressions}, new JoinType[] {JoinType.Left}, postJoinFilterExpression); HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); BasicQueryPlan lhsPlan = compileSingleQuery(context, lhs, binds); return new HashJoinPlan(lhsPlan, joinIds, new List[] {hashExpressions}, new QueryPlan[] {rhsPlan}); diff --git a/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java b/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java index 888ea348..8dd45f92 100644 --- a/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java +++ b/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java @@ -39,8 +39,10 @@ import com.salesforce.phoenix.cache.GlobalCache; import com.salesforce.phoenix.cache.TenantCache; +import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.join.HashJoinInfo; import com.salesforce.phoenix.parse.JoinTableNode.JoinType; +import com.salesforce.phoenix.schema.IllegalDataException; import com.salesforce.phoenix.schema.tuple.ResultTuple; import com.salesforce.phoenix.schema.tuple.Tuple; import com.salesforce.phoenix.util.TupleUtil; @@ -122,6 +124,27 @@ private void processResults(List result, boolean hasLimit) throws IOEx } } } + // apply post-join filter + Expression postFilter = joinInfo.getPostJoinFilterExpression(); + if (postFilter != null) { + ImmutableBytesWritable tempPtr = new ImmutableBytesWritable(); + for (Iterator> iter = resultQueue.iterator(); iter.hasNext();) { + Tuple t = new ResultTuple(new Result(iter.next())); + try { + if (!postFilter.evaluate(t, tempPtr)) { + iter.remove(); + continue; + } + } catch (IllegalDataException e) { + iter.remove(); + continue; + } + Boolean b = (Boolean)postFilter.getDataType().toObject(tempPtr); + if (!b.booleanValue()) { + iter.remove(); + } + } + } } } } diff --git a/src/main/java/com/salesforce/phoenix/join/HashJoinInfo.java b/src/main/java/com/salesforce/phoenix/join/HashJoinInfo.java index 14c1736f..5d281d98 100644 --- a/src/main/java/com/salesforce/phoenix/join/HashJoinInfo.java +++ b/src/main/java/com/salesforce/phoenix/join/HashJoinInfo.java @@ -45,11 +45,13 @@ public class HashJoinInfo { private ImmutableBytesWritable[] joinIds; private List[] joinExpressions; private JoinType[] joinTypes; + private Expression postJoinFilterExpression; - public HashJoinInfo(ImmutableBytesWritable[] joinIds, List[] joinExpressions, JoinType[] joinTypes) { + public HashJoinInfo(ImmutableBytesWritable[] joinIds, List[] joinExpressions, JoinType[] joinTypes, Expression postJoinFilterExpression) { this.joinIds = joinIds; this.joinExpressions = joinExpressions; this.joinTypes = joinTypes; + this.postJoinFilterExpression = postJoinFilterExpression; } public ImmutableBytesWritable[] getJoinIds() { @@ -64,6 +66,10 @@ public JoinType[] getJoinTypes() { return joinTypes; } + public Expression getPostJoinFilterExpression() { + return postJoinFilterExpression; + } + public static void serializeHashJoinIntoScan(Scan scan, HashJoinInfo joinInfo) { ByteArrayOutputStream stream = new ByteArrayOutputStream(); try { @@ -79,6 +85,12 @@ public static void serializeHashJoinIntoScan(Scan scan, HashJoinInfo joinInfo) { } WritableUtils.writeVInt(output, joinInfo.joinTypes[i].ordinal()); } + if (joinInfo.postJoinFilterExpression != null) { + WritableUtils.writeVInt(output, ExpressionType.valueOf(joinInfo.postJoinFilterExpression).ordinal()); + joinInfo.postJoinFilterExpression.write(output); + } else { + WritableUtils.writeVInt(output, -1); + } scan.setAttribute(HASH_JOIN, stream.toByteArray()); } catch (IOException e) { throw new RuntimeException(e); @@ -119,7 +131,13 @@ public static HashJoinInfo deserializeHashJoinFromScan(Scan scan) { int type = WritableUtils.readVInt(input); joinTypes[i] = JoinType.values()[type]; } - return new HashJoinInfo(joinIds, joinExpressions, joinTypes); + Expression postJoinFilterExpression = null; + int expressionOrdinal = WritableUtils.readVInt(input); + if (expressionOrdinal != -1) { + postJoinFilterExpression = ExpressionType.values()[expressionOrdinal].newInstance(); + postJoinFilterExpression.readFields(input); + } + return new HashJoinInfo(joinIds, joinExpressions, joinTypes, postJoinFilterExpression); } catch (IOException e) { throw new RuntimeException(e); } finally { From f130b9818f5679dee99d5afa2982852aab29e21f Mon Sep 17 00:00:00 2001 From: maryannxue Date: Mon, 19 Aug 2013 22:00:13 -0400 Subject: [PATCH 014/102] add subquery extraction implementation in JoinCompiler --- .../phoenix/compile/JoinCompiler.java | 108 +++++++++++++----- .../phoenix/compile/QueryCompiler.java | 4 +- 2 files changed, 84 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java index d7d6cf3d..ae7cdcc2 100644 --- a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java @@ -54,20 +54,22 @@ public enum StarJoinType { public static class JoinSpec { private TableRef mainTable; - private List select; - private ParseNode preFilters; - private List postFilters; + private List select; // all basic nodes related to mainTable, no aggregation. + private List preFilters; + private List postFilters; + private List postFilterExpressions; private List joinTables; private boolean isPostAggregateOrDistinct; private ColumnResolver resolver; - private JoinSpec(TableRef table, List select, ParseNode preFilters, - List postFilters, List joinTables, boolean isPostAggregate, - ColumnResolver resolver) { + private JoinSpec(TableRef table, List select, List preFilters, + List postFilters, List postFilterExpressions, + List joinTables, boolean isPostAggregate, ColumnResolver resolver) { this.mainTable = table; this.select = select; this.preFilters = preFilters; this.postFilters = postFilters; + this.postFilterExpressions = postFilterExpressions; this.joinTables = joinTables; this.isPostAggregateOrDistinct = isPostAggregate; this.resolver = resolver; @@ -81,14 +83,18 @@ public List getSelect() { return select; } - public ParseNode getPreFilters() { + public List getPreFilters() { return preFilters; } - public List getPostFilters() { + public List getPostFilters() { return postFilters; } + public List getPostFilterExpressions() { + return postFilterExpressions; + } + public List getJoinTables() { return joinTables; } @@ -100,10 +106,20 @@ public boolean isPostAggregateOrDistinct() { public ColumnResolver getColumnResolver() { return resolver; } + + public ParseNode getPreFiltersCombined() { + if (preFilters == null || preFilters.isEmpty()) + return null; + + if (preFilters.size() == 1) + return preFilters.get(0); + + return factory.and(preFilters); + } } public static JoinSpec getSubJoinSpec(JoinSpec join) { - return new JoinSpec(join.mainTable, join.select, join.preFilters, join.postFilters, join.joinTables.subList(0, join.joinTables.size() - 2), join.isPostAggregateOrDistinct, join.resolver); + return new JoinSpec(join.mainTable, join.select, join.preFilters, join.postFilters, join.postFilterExpressions, join.joinTables.subList(0, join.joinTables.size() - 2), join.isPostAggregateOrDistinct, join.resolver); } public static class JoinTable { @@ -111,19 +127,22 @@ public static class JoinTable { private List conditions; private TableNode tableNode; // original table node private TableRef table; - private List select; - private ParseNode preFilters; - private List postJoinFilters; // will be pushed to postFilters in case of star join + private List select; // all basic nodes related to this table, no aggregation. + private List preFilters; + private List postFilters; + private List postFilterExpressions; // will be pushed to postFilters in case of star join private SelectStatement subquery; private JoinTable(JoinType type, List conditions, TableNode tableNode, List select, - ParseNode preFilters, List postJoinFilters, TableRef table, SelectStatement subquery) { + List preFilters, List postFilters, List postFilterExpressions, + TableRef table, SelectStatement subquery) { this.type = type; this.conditions = conditions; this.tableNode = tableNode; this.select = select; this.preFilters = preFilters; - this.postJoinFilters = postJoinFilters; + this.postFilters = postFilters; + this.postFilterExpressions = postFilterExpressions; this.table = table; this.subquery = subquery; } @@ -148,18 +167,32 @@ public List getSelect() { return select; } - public ParseNode getPreFilters() { + public List getPreFilters() { return preFilters; } - public List getPostJoinFilters() { - return postJoinFilters; + public List getPostFilters() { + return postFilters; + } + + public List getPostFilterExpressions() { + return postFilterExpressions; } public SelectStatement getSubquery() { return subquery; } + public ParseNode getPreFiltersCombined() { + if (preFilters == null || preFilters.isEmpty()) + return null; + + if (preFilters.size() == 1) + return preFilters.get(0); + + return factory.and(preFilters); + } + public SelectStatement getAsSubquery() { // TODO return subquery; @@ -186,7 +219,7 @@ public static Pair, List> splitEquiJoinConditions(J } public static SelectStatement getSubqueryWithoutJoin(SelectStatement statement, JoinSpec join) { - return factory.select(statement.getFrom().subList(0, 1), statement.getHint(), statement.isDistinct(), statement.getSelect(), join.getPreFilters(), statement.getGroupBy(), statement.getHaving(), statement.getOrderBy(), statement.getLimit(), statement.getBindCount()); + return factory.select(statement.getFrom().subList(0, 1), statement.getHint(), statement.isDistinct(), statement.getSelect(), join.getPreFiltersCombined(), statement.getGroupBy(), statement.getHaving(), statement.getOrderBy(), statement.getLimit(), statement.getBindCount()); } // Get the last join table select statement with fixed-up select and where nodes. @@ -202,21 +235,44 @@ public static SelectStatement getSubqueryForLastJoinTable(SelectStatement statem List from = new ArrayList(1); from.add(lastJoinTable.getTableNode()); - // TODO distinguish last join in original query or last join in subquery - return factory.select(from, statement.getHint(), statement.isDistinct(), lastJoinTable.getSelect(), lastJoinTable.getPreFilters(), statement.getGroupBy(), statement.getHaving(), statement.getOrderBy(), statement.getLimit(), statement.getBindCount()); + return factory.select(from, statement.getHint(), statement.isDistinct(), statement.getSelect(), lastJoinTable.getPreFiltersCombined(), statement.getGroupBy(), statement.getHaving(), statement.getOrderBy(), statement.getLimit(), statement.getBindCount()); } // Get subquery with fixed select and where nodes - public static SelectStatement getSubQueryWithoutLastJoin(SelectStatement statement) { - // TODO - return null; + public static SelectStatement getSubQueryWithoutLastJoin(SelectStatement statement, JoinSpec join) { + List from = statement.getFrom(); + assert(from.size() > 1); + List joinTables = join.getJoinTables(); + int count = joinTables.size(); + assert (count > 0); + List select = new ArrayList(); + List filters = new ArrayList(); + select.addAll(join.getSelect()); + filters.addAll(join.getPreFilters()); + for (int i = 0; i < count - 1; i++) { + select.addAll(joinTables.get(i).getSelect()); + filters.addAll(joinTables.get(i).getPreFilters()); + filters.addAll(joinTables.get(i).getPostFilters()); + } + ParseNode where = null; + if (filters.size() == 1) { + where = filters.get(0); + } else if (filters.size() > 1) { + where = factory.and(filters); + } + + return factory.select(from.subList(0, from.size() - 1), statement.getHint(), statement.isDistinct(), select, where, null, null, null, null, statement.getBindCount()); } // Get subquery with complete select and where nodes // Throws exception if the subquery contains joins. - public static SelectStatement getSubQueryWithoutLastJoinAsFinalPlan(SelectStatement statement) { - // TODO - return null; + public static SelectStatement getSubQueryWithoutLastJoinAsFinalPlan(SelectStatement statement, JoinSpec join) { + List from = statement.getFrom(); + assert(from.size() > 1); + if (from.size() > 2) + throw new UnsupportedOperationException("Left table of a left join cannot contain joins."); + + return factory.select(from.subList(0, from.size() - 1), statement.getHint(), statement.isDistinct(), statement.getSelect(), join.getPreFiltersCombined(), statement.getGroupBy(), statement.getHaving(), statement.getOrderBy(), statement.getLimit(), statement.getBindCount()); } public static Expression getPostJoinFilterExpression(JoinSpec join, JoinTable joinTable) { diff --git a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java index 3e1e0b52..2758750c 100644 --- a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java @@ -178,7 +178,7 @@ protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement s if (type == JoinType.Right || (type == JoinType.Inner && joinTables.size() > 1)) { - SelectStatement lhs = JoinCompiler.getSubQueryWithoutLastJoin(statement); + SelectStatement lhs = JoinCompiler.getSubQueryWithoutLastJoin(statement, join); SelectStatement rhs = JoinCompiler.getSubqueryForLastJoinTable(statement, join); JoinSpec lhsJoin = JoinCompiler.getSubJoinSpec(join); StatementContext lhsCtx = new StatementContext(connection, join.getColumnResolver(), binds, statement.getBindCount(), scan, statement.getHint(), new HashCacheClient(connection.getQueryServices(), connection.getTenantId())); @@ -195,7 +195,7 @@ protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement s return new HashJoinPlan(rhsPlan, joinIds, new List[] {hashExpressions}, new QueryPlan[] {lhsPlan}); } - SelectStatement lhs = JoinCompiler.getSubQueryWithoutLastJoinAsFinalPlan(statement); + SelectStatement lhs = JoinCompiler.getSubQueryWithoutLastJoinAsFinalPlan(statement, join); SelectStatement rhs = lastJoinTable.getAsSubquery(); QueryPlan rhsPlan; try { From 5fbb3f20eefad21e903b81691ba80df6a5ad902d Mon Sep 17 00:00:00 2001 From: maryannxue Date: Mon, 26 Aug 2013 15:54:35 -0400 Subject: [PATCH 015/102] add JoinCompiler impl --- .../phoenix/compile/JoinCompiler.java | 87 ++++++++++++++----- .../phoenix/compile/QueryCompiler.java | 27 +++--- 2 files changed, 77 insertions(+), 37 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java index ae7cdcc2..25ebf225 100644 --- a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java @@ -31,8 +31,7 @@ import java.util.ArrayList; import java.util.List; -import org.apache.hadoop.hbase.util.Pair; - +import com.salesforce.phoenix.expression.AndExpression; import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.jdbc.PhoenixConnection; import com.salesforce.phoenix.parse.AliasedNode; @@ -114,7 +113,7 @@ public ParseNode getPreFiltersCombined() { if (preFilters.size() == 1) return preFilters.get(0); - return factory.and(preFilters); + return NODE_FACTORY.and(preFilters); } } @@ -132,6 +131,9 @@ public static class JoinTable { private List postFilters; private List postFilterExpressions; // will be pushed to postFilters in case of star join private SelectStatement subquery; + // equi-joins only + private List leftTableConditions; + private List rightTableConditions; private JoinTable(JoinType type, List conditions, TableNode tableNode, List select, List preFilters, List postFilters, List postFilterExpressions, @@ -145,6 +147,7 @@ private JoinTable(JoinType type, List conditions, TableNode tableNod this.postFilterExpressions = postFilterExpressions; this.table = table; this.subquery = subquery; + // TODO split left and right table conditions; } public JoinType getType() { @@ -190,17 +193,33 @@ public ParseNode getPreFiltersCombined() { if (preFilters.size() == 1) return preFilters.get(0); - return factory.and(preFilters); + return NODE_FACTORY.and(preFilters); } public SelectStatement getAsSubquery() { - // TODO - return subquery; + if (subquery != null) + return subquery; + + List from = new ArrayList(1); + from.add(tableNode); + return NODE_FACTORY.select(from, null, false, select, getPreFiltersCombined(), null, null, null, null, 0); + } + + public boolean isEquiJoin() { + return (leftTableConditions != null && !leftTableConditions.isEmpty()); + } + + public List getLeftTableConditions() { + return leftTableConditions; + } + + public List getRightTableConditions() { + return rightTableConditions; } } // for creation of new statements - private static ParseNodeFactory factory = new ParseNodeFactory(); + private static ParseNodeFactory NODE_FACTORY = new ParseNodeFactory(); public static JoinSpec getJoinSpec(SelectStatement statement, PhoenixConnection connection) throws SQLException { // TODO @@ -208,18 +227,27 @@ public static JoinSpec getJoinSpec(SelectStatement statement, PhoenixConnection } public static StarJoinType getStarJoinType(JoinSpec join) { - // TODO - return StarJoinType.NONE; - } - - // Left: other table expressions; Right: this table expressions. - public static Pair, List> splitEquiJoinConditions(JoinTable joinTable) { - // TODO - return null; + assert(!join.getJoinTables().isEmpty()); + + StarJoinType starJoinType = StarJoinType.BASIC; + for (JoinTable joinTable : join.getJoinTables()) { + if (!joinTable.isEquiJoin() + || (joinTable.getType() != JoinType.Left + && joinTable.getType() != JoinType.Inner)) + return StarJoinType.NONE; + if (starJoinType == StarJoinType.BASIC) { + for (Expression expr : joinTable.getLeftTableConditions()) { + // TODO test if expr consists ref to tables other than mainTable + starJoinType = StarJoinType.EXTENDED; + } + } + } + + return starJoinType; } public static SelectStatement getSubqueryWithoutJoin(SelectStatement statement, JoinSpec join) { - return factory.select(statement.getFrom().subList(0, 1), statement.getHint(), statement.isDistinct(), statement.getSelect(), join.getPreFiltersCombined(), statement.getGroupBy(), statement.getHaving(), statement.getOrderBy(), statement.getLimit(), statement.getBindCount()); + return NODE_FACTORY.select(statement.getFrom().subList(0, 1), statement.getHint(), statement.isDistinct(), statement.getSelect(), join.getPreFiltersCombined(), statement.getGroupBy(), statement.getHaving(), statement.getOrderBy(), statement.getLimit(), statement.getBindCount()); } // Get the last join table select statement with fixed-up select and where nodes. @@ -235,7 +263,7 @@ public static SelectStatement getSubqueryForLastJoinTable(SelectStatement statem List from = new ArrayList(1); from.add(lastJoinTable.getTableNode()); - return factory.select(from, statement.getHint(), statement.isDistinct(), statement.getSelect(), lastJoinTable.getPreFiltersCombined(), statement.getGroupBy(), statement.getHaving(), statement.getOrderBy(), statement.getLimit(), statement.getBindCount()); + return NODE_FACTORY.select(from, statement.getHint(), statement.isDistinct(), statement.getSelect(), lastJoinTable.getPreFiltersCombined(), statement.getGroupBy(), statement.getHaving(), statement.getOrderBy(), statement.getLimit(), statement.getBindCount()); } // Get subquery with fixed select and where nodes @@ -258,10 +286,10 @@ public static SelectStatement getSubQueryWithoutLastJoin(SelectStatement stateme if (filters.size() == 1) { where = filters.get(0); } else if (filters.size() > 1) { - where = factory.and(filters); + where = NODE_FACTORY.and(filters); } - return factory.select(from.subList(0, from.size() - 1), statement.getHint(), statement.isDistinct(), select, where, null, null, null, null, statement.getBindCount()); + return NODE_FACTORY.select(from.subList(0, from.size() - 1), statement.getHint(), statement.isDistinct(), select, where, null, null, null, null, statement.getBindCount()); } // Get subquery with complete select and where nodes @@ -272,11 +300,26 @@ public static SelectStatement getSubQueryWithoutLastJoinAsFinalPlan(SelectStatem if (from.size() > 2) throw new UnsupportedOperationException("Left table of a left join cannot contain joins."); - return factory.select(from.subList(0, from.size() - 1), statement.getHint(), statement.isDistinct(), statement.getSelect(), join.getPreFiltersCombined(), statement.getGroupBy(), statement.getHaving(), statement.getOrderBy(), statement.getLimit(), statement.getBindCount()); + return NODE_FACTORY.select(from.subList(0, from.size() - 1), statement.getHint(), statement.isDistinct(), statement.getSelect(), join.getPreFiltersCombined(), statement.getGroupBy(), statement.getHaving(), statement.getOrderBy(), statement.getLimit(), statement.getBindCount()); } public static Expression getPostJoinFilterExpression(JoinSpec join, JoinTable joinTable) { - // TODO - return null; + List postFilters = new ArrayList(); + if (joinTable != null) { + postFilters.addAll(joinTable.getPostFilterExpressions()); + } else { + for (JoinTable table : join.getJoinTables()) { + postFilters.addAll(table.getPostFilterExpressions()); + } + } + postFilters.addAll(join.getPostFilterExpressions()); + Expression postJoinFilterExpression = null; + if (postFilters.size() == 1) { + postJoinFilterExpression = postFilters.get(0); + } else if (postFilters.size() > 1) { + postJoinFilterExpression = new AndExpression(postFilters); + } + + return postJoinFilterExpression; } } diff --git a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java index 2758750c..123d1736 100644 --- a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java @@ -35,8 +35,6 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.util.Pair; - import com.salesforce.phoenix.compile.GroupByCompiler.GroupBy; import com.salesforce.phoenix.compile.JoinCompiler.JoinSpec; import com.salesforce.phoenix.compile.JoinCompiler.JoinTable; @@ -154,9 +152,10 @@ protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement s for (int i = 0; i < count; i++) { JoinTable joinTable = joinTables.get(i); joinIds[i] = new ImmutableBytesWritable(HashCacheClient.nextJoinId()); - Pair, List> splittedExpressions = JoinCompiler.splitEquiJoinConditions(joinTable); - joinExpressions[i] = splittedExpressions.getFirst(); - hashExpressions[i] = splittedExpressions.getSecond(); + if (!joinTable.isEquiJoin()) + throw new UnsupportedOperationException("Do not support non equi-joins."); + joinExpressions[i] = joinTable.getLeftTableConditions(); + hashExpressions[i] = joinTable.getRightTableConditions(); joinTypes[i] = joinTable.getType(); try { joinPlans[i] = compile(joinTable.getAsSubquery(), binds, new Scan(scanCopy)); @@ -185,14 +184,13 @@ protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement s QueryPlan lhsPlan = compileJoinQuery(lhsCtx, lhs, binds, lhsJoin); byte[] joinId = HashCacheClient.nextJoinId(); ImmutableBytesWritable[] joinIds = new ImmutableBytesWritable[] {new ImmutableBytesWritable(joinId)}; - Pair, List> splittedExpressions = JoinCompiler.splitEquiJoinConditions(lastJoinTable); - List joinExpressions = splittedExpressions.getSecond(); - List hashExpressions = splittedExpressions.getFirst(); + if (!lastJoinTable.isEquiJoin()) + throw new UnsupportedOperationException("Do not support non equi-joins."); Expression postJoinFilterExpression = JoinCompiler.getPostJoinFilterExpression(join, lastJoinTable); - HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {joinExpressions}, new JoinType[] {JoinType.Left}, postJoinFilterExpression); + HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {lastJoinTable.getRightTableConditions()}, new JoinType[] {JoinType.Left}, postJoinFilterExpression); HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); BasicQueryPlan rhsPlan = compileSingleQuery(context, rhs, binds); - return new HashJoinPlan(rhsPlan, joinIds, new List[] {hashExpressions}, new QueryPlan[] {lhsPlan}); + return new HashJoinPlan(rhsPlan, joinIds, new List[] {lastJoinTable.getLeftTableConditions()}, new QueryPlan[] {lhsPlan}); } SelectStatement lhs = JoinCompiler.getSubQueryWithoutLastJoinAsFinalPlan(statement, join); @@ -205,14 +203,13 @@ protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement s } byte[] joinId = HashCacheClient.nextJoinId(); ImmutableBytesWritable[] joinIds = new ImmutableBytesWritable[] {new ImmutableBytesWritable(joinId)}; - Pair, List> splittedExpressions = JoinCompiler.splitEquiJoinConditions(lastJoinTable); - List joinExpressions = splittedExpressions.getFirst(); - List hashExpressions = splittedExpressions.getSecond(); + if (!lastJoinTable.isEquiJoin()) + throw new UnsupportedOperationException("Do not support non equi-joins."); Expression postJoinFilterExpression = JoinCompiler.getPostJoinFilterExpression(join, null); - HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {joinExpressions}, new JoinType[] {JoinType.Left}, postJoinFilterExpression); + HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {lastJoinTable.getLeftTableConditions()}, new JoinType[] {JoinType.Left}, postJoinFilterExpression); HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); BasicQueryPlan lhsPlan = compileSingleQuery(context, lhs, binds); - return new HashJoinPlan(lhsPlan, joinIds, new List[] {hashExpressions}, new QueryPlan[] {rhsPlan}); + return new HashJoinPlan(lhsPlan, joinIds, new List[] {lastJoinTable.getRightTableConditions()}, new QueryPlan[] {rhsPlan}); } protected BasicQueryPlan compileSingleQuery(StatementContext context, SelectStatement statement, List binds) throws SQLException{ From 80d2a5a808d7cbe8e80c3aa6772bcbb85d7c16b6 Mon Sep 17 00:00:00 2001 From: maryannxue Date: Mon, 2 Sep 2013 23:23:00 -0400 Subject: [PATCH 016/102] add scan projection implementation --- .../phoenix/compile/JoinCompiler.java | 106 ++++++++++++------ .../phoenix/compile/QueryCompiler.java | 48 ++++---- .../phoenix/compile/StatementContext.java | 10 +- .../phoenix/coprocessor/ScanProjector.java | 6 +- 4 files changed, 108 insertions(+), 62 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java index cfed3cb3..4eb90654 100644 --- a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java @@ -31,6 +31,10 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hbase.util.Bytes; + +import com.salesforce.phoenix.coprocessor.ScanProjector; +import com.salesforce.phoenix.coprocessor.ScanProjector.ProjectionType; import com.salesforce.phoenix.expression.AndExpression; import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.jdbc.PhoenixConnection; @@ -41,6 +45,8 @@ import com.salesforce.phoenix.parse.TableNode; import com.salesforce.phoenix.parse.JoinTableNode.JoinType; import com.salesforce.phoenix.schema.TableRef; +import com.salesforce.phoenix.util.ByteUtil; +import com.salesforce.phoenix.util.SchemaUtil; public class JoinCompiler { @@ -56,19 +62,17 @@ public static class JoinSpec { private List select; // all basic nodes related to mainTable, no aggregation. private List preFilters; private List postFilters; - private List postFilterExpressions; private List joinTables; private boolean isPostAggregateOrDistinct; private ColumnResolver resolver; private JoinSpec(TableRef table, List select, List preFilters, - List postFilters, List postFilterExpressions, - List joinTables, boolean isPostAggregate, ColumnResolver resolver) { + List postFilters, List joinTables, boolean isPostAggregate, + ColumnResolver resolver) { this.mainTable = table; this.select = select; this.preFilters = preFilters; this.postFilters = postFilters; - this.postFilterExpressions = postFilterExpressions; this.joinTables = joinTables; this.isPostAggregateOrDistinct = isPostAggregate; this.resolver = resolver; @@ -90,10 +94,6 @@ public List getPostFilters() { return postFilters; } - public List getPostFilterExpressions() { - return postFilterExpressions; - } - public List getJoinTables() { return joinTables; } @@ -115,46 +115,55 @@ public ParseNode getPreFiltersCombined() { return NODE_FACTORY.and(preFilters); } + + public ScanProjector getScanProjector() { + byte[] tableName = null; + if (mainTable.getTableAlias() != null) { + tableName = Bytes.toBytes(mainTable.getTableAlias()); + } else { + tableName = mainTable.getTableName(); + } + byte[] tablePrefix = ByteUtil.concat(tableName, PROJECTION_SEPERATOR); + return new ScanProjector(ProjectionType.TABLE, tablePrefix, null, null); + } + + public List compilePostFilterExpressions() { + // TODO + return null; + } } public static JoinSpec getSubJoinSpec(JoinSpec join) { - return new JoinSpec(join.mainTable, join.select, join.preFilters, join.postFilters, join.postFilterExpressions, join.joinTables.subList(0, join.joinTables.size() - 2), join.isPostAggregateOrDistinct, join.resolver); + return new JoinSpec(join.mainTable, join.select, join.preFilters, join.postFilters, join.joinTables.subList(0, join.joinTables.size() - 2), join.isPostAggregateOrDistinct, join.resolver); } public static class JoinTable { private JoinType type; - private List conditions; + private List conditions; private TableNode tableNode; // original table node private TableRef table; private List select; // all basic nodes related to this table, no aggregation. private List preFilters; private List postFilters; - private List postFilterExpressions; // will be pushed to postFilters in case of star join private SelectStatement subquery; - // equi-joins only - private List leftTableConditions; - private List rightTableConditions; - private JoinTable(JoinType type, List conditions, TableNode tableNode, List select, - List preFilters, List postFilters, List postFilterExpressions, - TableRef table, SelectStatement subquery) { + private JoinTable(JoinType type, List conditions, TableNode tableNode, List select, + List preFilters, List postFilters, TableRef table, SelectStatement subquery) { this.type = type; this.conditions = conditions; this.tableNode = tableNode; this.select = select; this.preFilters = preFilters; this.postFilters = postFilters; - this.postFilterExpressions = postFilterExpressions; this.table = table; this.subquery = subquery; - // TODO split left and right table conditions; } public JoinType getType() { return type; } - public List getJoinConditions() { + public List getJoinConditions() { return conditions; } @@ -178,10 +187,6 @@ public List getPostFilters() { return postFilters; } - public List getPostFilterExpressions() { - return postFilterExpressions; - } - public SelectStatement getSubquery() { return subquery; } @@ -205,22 +210,44 @@ public SelectStatement getAsSubquery() { return NODE_FACTORY.select(from, null, false, select, getPreFiltersCombined(), null, null, null, null, 0, false); } - public boolean isEquiJoin() { - return (leftTableConditions != null && !leftTableConditions.isEmpty()); + public ScanProjector getScanProjector() { + byte[] tableName = null; + if (table.getTableAlias() != null) { + tableName = Bytes.toBytes(table.getTableAlias()); + } else { + tableName = table.getTableName(); + } + byte[] tablePrefix = ByteUtil.concat(tableName, PROJECTION_SEPERATOR); + return new ScanProjector(ProjectionType.TABLE, tablePrefix, null, null); } - public List getLeftTableConditions() { - return leftTableConditions; + public List compilePostFilterExpressions() { + // TODO + return null; } - public List getRightTableConditions() { - return rightTableConditions; + /** + * @throws SQLException if it is not an equi-join. + */ + public List compileLeftTableConditions() throws SQLException { + // TODO + return null; + } + + /** + * @throws SQLException if it is not an equi-join. + */ + public List compileRightTableConditions() throws SQLException { + // TODO + return null; } } // for creation of new statements private static ParseNodeFactory NODE_FACTORY = new ParseNodeFactory(); + private static final byte[] PROJECTION_SEPERATOR = Bytes.toBytes(":"); + public static JoinSpec getJoinSpec(SelectStatement statement, PhoenixConnection connection) throws SQLException { // TODO return null; @@ -231,12 +258,17 @@ public static StarJoinType getStarJoinType(JoinSpec join) { StarJoinType starJoinType = StarJoinType.BASIC; for (JoinTable joinTable : join.getJoinTables()) { - if (!joinTable.isEquiJoin() - || (joinTable.getType() != JoinType.Left - && joinTable.getType() != JoinType.Inner)) + if (joinTable.getType() != JoinType.Left + && joinTable.getType() != JoinType.Inner) return StarJoinType.NONE; if (starJoinType == StarJoinType.BASIC) { - for (Expression expr : joinTable.getLeftTableConditions()) { + List leftTableConditions; + try { + leftTableConditions = joinTable.compileLeftTableConditions(); + } catch (SQLException e) { + return StarJoinType.NONE; + } + for (Expression expr : leftTableConditions) { // TODO test if expr consists ref to tables other than mainTable starJoinType = StarJoinType.EXTENDED; } @@ -306,13 +338,13 @@ public static SelectStatement getSubQueryWithoutLastJoinAsFinalPlan(SelectStatem public static Expression getPostJoinFilterExpression(JoinSpec join, JoinTable joinTable) { List postFilters = new ArrayList(); if (joinTable != null) { - postFilters.addAll(joinTable.getPostFilterExpressions()); + postFilters.addAll(joinTable.compilePostFilterExpressions()); } else { for (JoinTable table : join.getJoinTables()) { - postFilters.addAll(table.getPostFilterExpressions()); + postFilters.addAll(table.compilePostFilterExpressions()); } } - postFilters.addAll(join.getPostFilterExpressions()); + postFilters.addAll(join.compilePostFilterExpressions()); Expression postJoinFilterExpression = null; if (postFilters.size() == 1) { postJoinFilterExpression = postFilters.get(0); diff --git a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java index 6910142e..0e964479 100644 --- a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java @@ -39,6 +39,7 @@ import com.salesforce.phoenix.compile.JoinCompiler.JoinTable; import com.salesforce.phoenix.compile.JoinCompiler.StarJoinType; import com.salesforce.phoenix.compile.OrderByCompiler.OrderBy; +import com.salesforce.phoenix.coprocessor.ScanProjector; import com.salesforce.phoenix.execute.*; import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.iterate.ParallelIterators.ParallelIteratorFactory; @@ -130,20 +131,22 @@ protected QueryPlan compile(SelectStatement statement, List binds, Scan List fromNodes = statement.getFrom(); if (fromNodes.size() == 1) { ColumnResolver resolver = FromCompiler.getResolver(statement, connection); - StatementContext context = new StatementContext(statement, connection, resolver, binds, scan, null); + StatementContext context = new StatementContext(statement, connection, resolver, binds, scan); return compileSingleQuery(context, statement, binds); } JoinSpec join = JoinCompiler.getJoinSpec(statement, connection); - StatementContext context = new StatementContext(statement, connection, join.getColumnResolver(), binds, scan, new HashCacheClient(connection)); + StatementContext context = new StatementContext(statement, connection, join.getColumnResolver(), binds, scan, true, new HashCacheClient(connection)); return compileJoinQuery(context, statement, binds, join); } @SuppressWarnings("unchecked") protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement statement, List binds, JoinSpec join) throws SQLException { List joinTables = join.getJoinTables(); - if (joinTables.isEmpty()) + if (joinTables.isEmpty()) { + ScanProjector.serializeProjectorIntoScan(context.getScan(), join.getScanProjector()); return compileSingleQuery(context, statement, binds); + } StarJoinType starJoin = JoinCompiler.getStarJoinType(join); if (starJoin == StarJoinType.BASIC || starJoin == StarJoinType.EXTENDED) { @@ -156,13 +159,13 @@ protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement s for (int i = 0; i < count; i++) { JoinTable joinTable = joinTables.get(i); joinIds[i] = new ImmutableBytesPtr(); // place-holder - if (!joinTable.isEquiJoin()) - throw new UnsupportedOperationException("Do not support non equi-joins."); - joinExpressions[i] = joinTable.getLeftTableConditions(); - hashExpressions[i] = joinTable.getRightTableConditions(); + joinExpressions[i] = joinTable.compileLeftTableConditions(); + hashExpressions[i] = joinTable.compileRightTableConditions(); joinTypes[i] = joinTable.getType(); try { - joinPlans[i] = compile(joinTable.getAsSubquery(), binds, new Scan(scanCopy)); + Scan subScan = new Scan(scanCopy); + ScanProjector.serializeProjectorIntoScan(subScan, joinTable.getScanProjector()); + joinPlans[i] = compile(joinTable.getAsSubquery(), binds, subScan); } catch (IOException e) { throw new SQLException(e); } @@ -170,6 +173,7 @@ protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement s Expression postJoinFilterExpression = JoinCompiler.getPostJoinFilterExpression(join, null); HashJoinInfo joinInfo = new HashJoinInfo(joinIds, joinExpressions, joinTypes, postJoinFilterExpression); HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); + ScanProjector.serializeProjectorIntoScan(context.getScan(), join.getScanProjector()); BasicQueryPlan plan = compileSingleQuery(context, JoinCompiler.getSubqueryWithoutJoin(statement, join), binds); return new HashJoinPlan(plan, joinIds, hashExpressions, joinPlans); } @@ -184,34 +188,40 @@ protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement s SelectStatement lhs = JoinCompiler.getSubQueryWithoutLastJoin(statement, join); SelectStatement rhs = JoinCompiler.getSubqueryForLastJoinTable(statement, join); JoinSpec lhsJoin = JoinCompiler.getSubJoinSpec(join); - StatementContext lhsCtx = new StatementContext(statement, connection, join.getColumnResolver(), binds, scan, context.getHashClient()); + Scan subScan; + try { + subScan = new Scan(scanCopy); + } catch (IOException e) { + throw new SQLException(e); + } + StatementContext lhsCtx = new StatementContext(statement, connection, join.getColumnResolver(), binds, subScan, true, context.getHashClient()); QueryPlan lhsPlan = compileJoinQuery(lhsCtx, lhs, binds, lhsJoin); ImmutableBytesPtr[] joinIds = new ImmutableBytesPtr[] {new ImmutableBytesPtr()}; - if (!lastJoinTable.isEquiJoin()) - throw new UnsupportedOperationException("Do not support non equi-joins."); Expression postJoinFilterExpression = JoinCompiler.getPostJoinFilterExpression(join, lastJoinTable); - HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {lastJoinTable.getRightTableConditions()}, new JoinType[] {JoinType.Left}, postJoinFilterExpression); + HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {lastJoinTable.compileRightTableConditions()}, new JoinType[] {JoinType.Left}, postJoinFilterExpression); HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); + ScanProjector.serializeProjectorIntoScan(context.getScan(), lastJoinTable.getScanProjector()); BasicQueryPlan rhsPlan = compileSingleQuery(context, rhs, binds); - return new HashJoinPlan(rhsPlan, joinIds, new List[] {lastJoinTable.getLeftTableConditions()}, new QueryPlan[] {lhsPlan}); + return new HashJoinPlan(rhsPlan, joinIds, new List[] {lastJoinTable.compileLeftTableConditions()}, new QueryPlan[] {lhsPlan}); } SelectStatement lhs = JoinCompiler.getSubQueryWithoutLastJoinAsFinalPlan(statement, join); SelectStatement rhs = lastJoinTable.getAsSubquery(); - QueryPlan rhsPlan; + Scan subScan; try { - rhsPlan = compile(rhs, binds, new Scan(scanCopy)); + subScan = new Scan(scanCopy); } catch (IOException e) { throw new SQLException(e); } + ScanProjector.serializeProjectorIntoScan(subScan, lastJoinTable.getScanProjector()); + QueryPlan rhsPlan = compile(rhs, binds, subScan); ImmutableBytesPtr[] joinIds = new ImmutableBytesPtr[] {new ImmutableBytesPtr()}; - if (!lastJoinTable.isEquiJoin()) - throw new UnsupportedOperationException("Do not support non equi-joins."); Expression postJoinFilterExpression = JoinCompiler.getPostJoinFilterExpression(join, null); - HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {lastJoinTable.getLeftTableConditions()}, new JoinType[] {JoinType.Left}, postJoinFilterExpression); + HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {lastJoinTable.compileLeftTableConditions()}, new JoinType[] {JoinType.Left}, postJoinFilterExpression); HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); + ScanProjector.serializeProjectorIntoScan(context.getScan(), join.getScanProjector()); BasicQueryPlan lhsPlan = compileSingleQuery(context, lhs, binds); - return new HashJoinPlan(lhsPlan, joinIds, new List[] {lastJoinTable.getRightTableConditions()}, new QueryPlan[] {rhsPlan}); + return new HashJoinPlan(lhsPlan, joinIds, new List[] {lastJoinTable.compileRightTableConditions()}, new QueryPlan[] {rhsPlan}); } protected BasicQueryPlan compileSingleQuery(StatementContext context, SelectStatement statement, List binds) throws SQLException{ diff --git a/src/main/java/com/salesforce/phoenix/compile/StatementContext.java b/src/main/java/com/salesforce/phoenix/compile/StatementContext.java index 4b27018f..5333bfb0 100644 --- a/src/main/java/com/salesforce/phoenix/compile/StatementContext.java +++ b/src/main/java/com/salesforce/phoenix/compile/StatementContext.java @@ -69,13 +69,14 @@ public class StatementContext { private long currentTime = QueryConstants.UNSET_TIMESTAMP; private ScanRanges scanRanges = ScanRanges.EVERYTHING; + private final boolean prefixColumnFamily; private final HashCacheClient hashClient; public StatementContext(BindableStatement statement, PhoenixConnection connection, ColumnResolver resolver, List binds, Scan scan) { - this(statement, connection, resolver, binds, scan, null); + this(statement, connection, resolver, binds, scan, false, null); } - public StatementContext(BindableStatement statement, PhoenixConnection connection, ColumnResolver resolver, List binds, Scan scan, HashCacheClient hashClient) { + public StatementContext(BindableStatement statement, PhoenixConnection connection, ColumnResolver resolver, List binds, Scan scan, boolean prefixColumnFamily, HashCacheClient hashClient) { this.connection = connection; this.resolver = resolver; this.scan = scan; @@ -87,6 +88,7 @@ public StatementContext(BindableStatement statement, PhoenixConnection connectio this.dateParser = DateUtil.getDateParser(dateFormat); this.numberFormat = connection.getQueryServices().getProps().get(QueryServices.NUMBER_FORMAT_ATTRIB, NumberUtil.DEFAULT_NUMBER_FORMAT); this.tempPtr = new ImmutableBytesWritable(); + this.prefixColumnFamily = prefixColumnFamily; this.hashClient = hashClient; } @@ -114,6 +116,10 @@ public BindManager getBindManager() { return binds; } + public boolean shouldPrefixColumnFamily() { + return prefixColumnFamily; + } + public HashCacheClient getHashClient() { return hashClient; } diff --git a/src/main/java/com/salesforce/phoenix/coprocessor/ScanProjector.java b/src/main/java/com/salesforce/phoenix/coprocessor/ScanProjector.java index 4ada2fd7..7c20b22e 100644 --- a/src/main/java/com/salesforce/phoenix/coprocessor/ScanProjector.java +++ b/src/main/java/com/salesforce/phoenix/coprocessor/ScanProjector.java @@ -37,7 +37,6 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.io.WritableUtils; @@ -50,14 +49,13 @@ public class ScanProjector { public enum ProjectionType {TABLE, CF, CQ}; private static final String SCAN_PROJECTOR = "scanProjector"; - private static final byte[] SEPERATOR = Bytes.toBytes(":"); private final ProjectionType type; private final byte[] tablePrefix; private final Map cfProjectionMap; private final Map>> cqProjectionMap; - private ScanProjector(ProjectionType type, byte[] tablePrefix, + public ScanProjector(ProjectionType type, byte[] tablePrefix, Map cfProjectionMap, Map>> cqProjectionMap) { this.type = ProjectionType.TABLE; @@ -183,7 +181,7 @@ public Map>> getC public KeyValue getProjectedKeyValue(KeyValue kv) { if (type == ProjectionType.TABLE) { - byte[] cf = ByteUtil.concat(tablePrefix, SEPERATOR, kv.getFamily()); + byte[] cf = ByteUtil.concat(tablePrefix, kv.getFamily()); return KeyValueUtil.newKeyValue(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength(), cf, kv.getQualifier(), kv.getTimestamp(), kv.getBuffer(), kv.getValueOffset(), kv.getValueLength()); } From 670a35828e0772347a5b54c4d9d58dca8cdd559d Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 13 Sep 2013 08:21:16 -0700 Subject: [PATCH 017/102] Set info ports to -1 --- .../com/salesforce/hbase/index/IndexTestingUtils.java | 8 ++++++++ .../covered/TestEndToEndCoveredColumnsIndexBuilder.java | 4 ++-- .../covered/example/TestEndToEndCoveredIndexing.java | 1 + .../regionserver/wal/TestReadWriteKeyValuesWithCodec.java | 2 ++ .../regionserver/wal/TestWALReplayWithIndexWrites.java | 2 ++ 5 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/salesforce/hbase/index/IndexTestingUtils.java b/src/test/java/com/salesforce/hbase/index/IndexTestingUtils.java index d5f5c9fa..41499302 100644 --- a/src/test/java/com/salesforce/hbase/index/IndexTestingUtils.java +++ b/src/test/java/com/salesforce/hbase/index/IndexTestingUtils.java @@ -35,6 +35,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.HTable; @@ -50,10 +51,17 @@ public class IndexTestingUtils { private static final Log LOG = LogFactory.getLog(IndexTestingUtils.class); + private static final String MASTER_INFO_PORT_KEY = "hbase.master.info.port"; + private static final String RS_INFO_PORT_KEY = "hbase.regionserver.info.port"; + private IndexTestingUtils() { // private ctor for util class } + public static void setupConfig(Configuration conf) { + conf.setInt(MASTER_INFO_PORT_KEY, -1); + conf.setInt(RS_INFO_PORT_KEY, -1); + } /** * Verify the state of the index table between the given key and time ranges against the list of * expected keyvalues. diff --git a/src/test/java/com/salesforce/hbase/index/covered/TestEndToEndCoveredColumnsIndexBuilder.java b/src/test/java/com/salesforce/hbase/index/covered/TestEndToEndCoveredColumnsIndexBuilder.java index 2c1773e2..8d41bff3 100644 --- a/src/test/java/com/salesforce/hbase/index/covered/TestEndToEndCoveredColumnsIndexBuilder.java +++ b/src/test/java/com/salesforce/hbase/index/covered/TestEndToEndCoveredColumnsIndexBuilder.java @@ -57,10 +57,9 @@ import org.junit.Rule; import org.junit.Test; +import com.salesforce.hbase.index.IndexTestingUtils; import com.salesforce.hbase.index.Indexer; import com.salesforce.hbase.index.TableName; -import com.salesforce.hbase.index.covered.CoveredColumnsIndexBuilder; -import com.salesforce.hbase.index.covered.IndexCodec; import com.salesforce.hbase.index.covered.update.ColumnReference; import com.salesforce.hbase.index.scanner.Scanner; @@ -104,6 +103,7 @@ public TestState(HTable primary, VerifyingIndexCodec codec, long ts) { @BeforeClass public static void setupCluster() throws Exception { Configuration conf = UTIL.getConfiguration(); + IndexTestingUtils.setupConfig(conf); // disable version checking, so we can test against whatever version of HBase happens to be // installed (right now, its generally going to be SNAPSHOT versions). conf.setBoolean(Indexer.CHECK_VERSION_CONF_KEY, false); diff --git a/src/test/java/com/salesforce/hbase/index/covered/example/TestEndToEndCoveredIndexing.java b/src/test/java/com/salesforce/hbase/index/covered/example/TestEndToEndCoveredIndexing.java index a15384b9..9453b80f 100644 --- a/src/test/java/com/salesforce/hbase/index/covered/example/TestEndToEndCoveredIndexing.java +++ b/src/test/java/com/salesforce/hbase/index/covered/example/TestEndToEndCoveredIndexing.java @@ -110,6 +110,7 @@ private String getIndexTableName() { @BeforeClass public static void setupCluster() throws Exception { Configuration conf = UTIL.getConfiguration(); + IndexTestingUtils.setupConfig(conf); // disable version checking, so we can test against whatever version of HBase happens to be // installed (right now, its generally going to be SNAPSHOT versions). conf.setBoolean(Indexer.CHECK_VERSION_CONF_KEY, false); diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestReadWriteKeyValuesWithCodec.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestReadWriteKeyValuesWithCodec.java index d560baf8..b598e9bc 100644 --- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestReadWriteKeyValuesWithCodec.java +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestReadWriteKeyValuesWithCodec.java @@ -20,6 +20,7 @@ import org.junit.BeforeClass; import org.junit.Test; +import com.salesforce.hbase.index.IndexTestingUtils; import com.salesforce.hbase.index.wal.IndexedKeyValue; /** @@ -35,6 +36,7 @@ public class TestReadWriteKeyValuesWithCodec { @BeforeClass public static void setupCodec() { Configuration conf = UTIL.getConfiguration(); + IndexTestingUtils.setupConfig(conf); conf.set(WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY, IndexedWALEditCodec.class.getName()); } diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWrites.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWrites.java index 2a185bca..e445ab26 100644 --- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWrites.java +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWrites.java @@ -37,6 +37,7 @@ import org.junit.Test; import org.mockito.Mockito; +import com.salesforce.hbase.index.IndexTestingUtils; import com.salesforce.hbase.index.builder.example.ColumnFamilyIndexer; /** @@ -64,6 +65,7 @@ protected static void configureCluster() throws Exception { conf.setInt("zookeeper.recovery.retry.intervalmill", 100); conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 30000); conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 5000); + IndexTestingUtils.setupConfig(conf); // enable appends conf.setBoolean("dfs.support.append", true); From aac29ba29802756759f221300db15504b533ffe3 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 13 Sep 2013 10:25:39 -0700 Subject: [PATCH 018/102] Setting info port to -1 in a few missing places in secondary index tests, decreasing TTL of metadata cache since it can lead to issue #417 --- .../com/salesforce/phoenix/query/QueryServicesOptions.java | 2 +- .../java/com/salesforce/hbase/index/TestEndtoEndIndexing.java | 3 +++ .../hbase/index/TestFailForUnsupportedHBaseVersions.java | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/salesforce/phoenix/query/QueryServicesOptions.java b/src/main/java/com/salesforce/phoenix/query/QueryServicesOptions.java index 978e68af..2d7aefb2 100644 --- a/src/main/java/com/salesforce/phoenix/query/QueryServicesOptions.java +++ b/src/main/java/com/salesforce/phoenix/query/QueryServicesOptions.java @@ -95,7 +95,7 @@ public class QueryServicesOptions { public final static int DEFAULT_MUTATE_BATCH_SIZE = 1000; // Batch size for UPSERT SELECT and DELETE // The only downside of it being out-of-sync is that the parallelization of the scan won't be as balanced as it could be. - public static final int DEFAULT_REGION_BOUNDARY_CACHE_TTL_MS = 60000; // How long to cache region boundary info for parallelization calculation + public static final int DEFAULT_REGION_BOUNDARY_CACHE_TTL_MS = 15000; // How long to cache region boundary info for parallelization calculation public static final int DEFAULT_MAX_SERVER_CACHE_TIME_TO_LIVE_MS = 30000; // 30 sec (with no activity) public static final int DEFAULT_SCAN_CACHE_SIZE = 1000; public static final int DEFAULT_MAX_INTRA_REGION_PARALLELIZATION = DEFAULT_MAX_QUERY_CONCURRENCY; diff --git a/src/test/java/com/salesforce/hbase/index/TestEndtoEndIndexing.java b/src/test/java/com/salesforce/hbase/index/TestEndtoEndIndexing.java index 302cd42b..a5711855 100644 --- a/src/test/java/com/salesforce/hbase/index/TestEndtoEndIndexing.java +++ b/src/test/java/com/salesforce/hbase/index/TestEndtoEndIndexing.java @@ -37,6 +37,7 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; @@ -68,6 +69,8 @@ public class TestEndtoEndIndexing { @BeforeClass public static void setupCluster() throws Exception { + Configuration conf = UTIL.getConfiguration(); + IndexTestingUtils.setupConfig(conf); UTIL.startMiniCluster(); } diff --git a/src/test/java/com/salesforce/hbase/index/TestFailForUnsupportedHBaseVersions.java b/src/test/java/com/salesforce/hbase/index/TestFailForUnsupportedHBaseVersions.java index 5784507e..97e6b552 100644 --- a/src/test/java/com/salesforce/hbase/index/TestFailForUnsupportedHBaseVersions.java +++ b/src/test/java/com/salesforce/hbase/index/TestFailForUnsupportedHBaseVersions.java @@ -63,6 +63,7 @@ public class TestFailForUnsupportedHBaseVersions { @Test public void testDoesNotSupportCompressedWAL() { Configuration conf = HBaseConfiguration.create(); + IndexTestingUtils.setupConfig(conf); // get the current version String version = VersionInfo.getVersion(); @@ -111,6 +112,7 @@ public void testDoesNotSupportCompressedWAL() { @Test(timeout = 300000 /* 5 mins */) public void testDoesNotStartRegionServerForUnsupportedCompressionAndVersion() throws Exception { Configuration conf = HBaseConfiguration.create(); + IndexTestingUtils.setupConfig(conf); // enable WAL Compression conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true); From 1d431fbc547693e5383f12077689a83746461cc2 Mon Sep 17 00:00:00 2001 From: Matt Walsh Date: Fri, 13 Sep 2013 13:10:15 -0500 Subject: [PATCH 019/102] Adding new MD5Function based off MessageDigest for varchar. --- .../phoenix/expression/ExpressionType.java | 1 + .../expression/function/MD5Function.java | 65 +++++++++++++++++++ .../phoenix/end2end/MD5FunctionTest.java | 34 ++++++++++ 3 files changed, 100 insertions(+) create mode 100644 src/main/java/com/salesforce/phoenix/expression/function/MD5Function.java create mode 100644 src/test/java/com/salesforce/phoenix/end2end/MD5FunctionTest.java diff --git a/src/main/java/com/salesforce/phoenix/expression/ExpressionType.java b/src/main/java/com/salesforce/phoenix/expression/ExpressionType.java index a18d8d18..2181cfd0 100644 --- a/src/main/java/com/salesforce/phoenix/expression/ExpressionType.java +++ b/src/main/java/com/salesforce/phoenix/expression/ExpressionType.java @@ -42,6 +42,7 @@ * @since 0.1 */ public enum ExpressionType { + MD5Function(MD5Function.class), ReverseFunction(ReverseFunction.class), RowKey(RowKeyColumnExpression.class), KeyValue(KeyValueColumnExpression.class), diff --git a/src/main/java/com/salesforce/phoenix/expression/function/MD5Function.java b/src/main/java/com/salesforce/phoenix/expression/function/MD5Function.java new file mode 100644 index 00000000..2b1e80bc --- /dev/null +++ b/src/main/java/com/salesforce/phoenix/expression/function/MD5Function.java @@ -0,0 +1,65 @@ +package com.salesforce.phoenix.expression.function; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.sql.SQLException; +import java.util.List; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; + +import com.salesforce.phoenix.expression.Expression; +import com.salesforce.phoenix.parse.FunctionParseNode; +import com.salesforce.phoenix.schema.PDataType; +import com.salesforce.phoenix.schema.tuple.Tuple; + +@FunctionParseNode.BuiltInFunction(name=MD5Function.NAME, args={ + @FunctionParseNode.Argument(allowedTypes={PDataType.VARCHAR})} ) +public class MD5Function extends ScalarFunction { + public static final String NAME = "MD5"; + + public MD5Function() { + } + + public MD5Function(List children) throws SQLException { + super(children); + } + + @Override + public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { + if (!getStrExpression().evaluate(tuple, ptr)) { + return false; + } + + String sourceStr = (String)PDataType.VARCHAR.toObject(ptr, getStrExpression().getColumnModifier()); + + if (sourceStr == null) { + return true; + } + + try { + ptr.set(MessageDigest.getInstance("MD5").digest(sourceStr.getBytes())); + return true; + } catch (NoSuchAlgorithmException e) { + throw new IllegalArgumentException(e); + } + } + + @Override + public PDataType getDataType() { + return getStrExpression().getDataType(); + } + + @Override + public boolean isNullable() { + return getStrExpression().isNullable(); + } + + @Override + public String getName() { + return NAME; + } + + private Expression getStrExpression() { + return children.get(0); + } +} diff --git a/src/test/java/com/salesforce/phoenix/end2end/MD5FunctionTest.java b/src/test/java/com/salesforce/phoenix/end2end/MD5FunctionTest.java new file mode 100644 index 00000000..923f86c4 --- /dev/null +++ b/src/test/java/com/salesforce/phoenix/end2end/MD5FunctionTest.java @@ -0,0 +1,34 @@ +package com.salesforce.phoenix.end2end; + +import static org.junit.Assert.*; + +import java.security.MessageDigest; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; + +import org.junit.Test; + +public class MD5FunctionTest extends BaseHBaseManagedTimeTest { + + @Test + public void testReverse() throws Exception { + String testString = "mwalsh"; + + Connection conn = DriverManager.getConnection(getUrl()); + String ddl = "CREATE TABLE IF NOT EXISTS MD5_TEST (pk VARCHAR NOT NULL PRIMARY KEY)"; + conn.createStatement().execute(ddl); + String dml = String.format("UPSERT INTO MD5_TEST VALUES('%s')", testString); + conn.createStatement().execute(dml); + conn.commit(); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT MD5(pk) FROM MD5_TEST"); + assertTrue(rs.next()); + String first = new String(MessageDigest.getInstance("MD5").digest(testString.getBytes())); + String second = rs.getString(1); + assertEquals(first, second); + assertFalse(rs.next()); + } + +} From 5399dd4e3736d5a9cb88c69e35e30cc30d710db0 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 13 Sep 2013 12:03:35 -0700 Subject: [PATCH 020/102] Misc renames for consistency, added one more place to set info ports to -1 --- .../salesforce/phoenix/filter/SkipScanFilter.java | 4 ++-- .../salesforce/phoenix/index/IndexMaintainer.java | 4 ++-- .../salesforce/phoenix/iterate/ExplainTable.java | 2 +- .../salesforce/phoenix/join/HashCacheClient.java | 2 +- .../salesforce/phoenix/query/QueryServices.java | 2 +- .../phoenix/query/QueryServicesOptions.java | 10 +++++----- .../salesforce/phoenix/schema/KeyValueSchema.java | 4 ++-- .../salesforce/phoenix/schema/RowKeySchema.java | 14 +++++++------- .../com/salesforce/phoenix/schema/ValueSchema.java | 6 +++--- .../java/com/salesforce/phoenix/util/ScanUtil.java | 8 ++++---- .../index/TestEndtoEndIndexingWithCompression.java | 9 ++++++--- .../phoenix/query/QueryServicesTestImpl.java | 2 +- .../phoenix/schema/RowKeySchemaTest.java | 4 ++-- 13 files changed, 37 insertions(+), 34 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/filter/SkipScanFilter.java b/src/main/java/com/salesforce/phoenix/filter/SkipScanFilter.java index ebf240eb..4ac68213 100644 --- a/src/main/java/com/salesforce/phoenix/filter/SkipScanFilter.java +++ b/src/main/java/com/salesforce/phoenix/filter/SkipScanFilter.java @@ -421,7 +421,7 @@ private int setStartKey(ImmutableBytesWritable ptr, int offset, int i) { startKey = copyKey(startKey, length + this.maxKeyLength, ptr.get(), offset, length); startKeyLength = length; // Add separator byte if we're at the end of the buffer, since trailing separator bytes are stripped - if (ptr.getOffset() + ptr.getLength() == offset + length && i-1 > 0 && !schema.getField(i-1).getType().isFixedWidth()) { + if (ptr.getOffset() + ptr.getLength() == offset + length && i-1 > 0 && !schema.getField(i-1).getDataType().isFixedWidth()) { startKey[startKeyLength++] = QueryConstants.SEPARATOR_BYTE; } startKeyLength += setKey(Bound.LOWER, startKey, startKeyLength, i); @@ -455,7 +455,7 @@ private int getTerminatorCount(RowKeySchema schema) { // We won't have a terminator on the last PK column // unless it is variable length and exclusive, but // having the extra byte irregardless won't hurt anything - if (!field.getType().isFixedWidth()) { + if (!field.getDataType().isFixedWidth()) { nTerminators++; } } diff --git a/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java b/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java index e0801006..c5312209 100644 --- a/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java +++ b/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java @@ -240,7 +240,7 @@ public byte[] buildRowKey(ValueGetter valueGetter, ImmutableBytesWritable rowKey j++; } else { Field field = dataRowKeySchema.getField(dataPkPosition[i]); - dataColumnType = field.getType(); + dataColumnType = field.getDataType(); ptr.set(rowKeyPtr.get(), dataRowKeyLocator[0][i], dataRowKeyLocator[1][i]); dataColumnModifier = field.getColumnModifier(); isDataColumnInverted = dataColumnModifier != null; @@ -403,7 +403,7 @@ private void initCachedState() { dataType = indexedColumnTypes.get(indexedColumnTypesPos--); } else { Field dataField = dataRowKeySchema.getField(dataPkPos); - dataType = dataField.getType(); + dataType = dataField.getDataType(); isDataNullable = dataField.isNullable(); } PDataType indexDataType = IndexUtil.getIndexColumnDataType(isDataNullable, dataType); diff --git a/src/main/java/com/salesforce/phoenix/iterate/ExplainTable.java b/src/main/java/com/salesforce/phoenix/iterate/ExplainTable.java index edafb6f3..51af00eb 100644 --- a/src/main/java/com/salesforce/phoenix/iterate/ExplainTable.java +++ b/src/main/java/com/salesforce/phoenix/iterate/ExplainTable.java @@ -154,7 +154,7 @@ private void appendPKColumnValue(StringBuilder buf, byte[] range, int slotIndex) return; } ScanRanges scanRanges = context.getScanRanges(); - PDataType type = scanRanges.getSchema().getField(slotIndex).getType(); + PDataType type = scanRanges.getSchema().getField(slotIndex).getDataType(); ColumnModifier modifier = table.getTable().getPKColumns().get(slotIndex).getColumnModifier(); if (modifier != null) { range = modifier.apply(range, 0, new byte[range.length], 0, range.length); diff --git a/src/main/java/com/salesforce/phoenix/join/HashCacheClient.java b/src/main/java/com/salesforce/phoenix/join/HashCacheClient.java index b3b5b045..795f3164 100644 --- a/src/main/java/com/salesforce/phoenix/join/HashCacheClient.java +++ b/src/main/java/com/salesforce/phoenix/join/HashCacheClient.java @@ -92,7 +92,7 @@ public ServerCache addHashCache(ScanRanges keyRanges, Scanner scanner, List onExpressions) throws SQLException { - long maxSize = serverCache.getConnection().getQueryServices().getProps().getLong(QueryServices.MAX_HASH_CACHE_SIZE_ATTRIB, QueryServicesOptions.DEFAULT_MAX_HASH_CACHE_SIZE); + long maxSize = serverCache.getConnection().getQueryServices().getProps().getLong(QueryServices.MAX_SERVER_CACHE_SIZE_ATTRIB, QueryServicesOptions.DEFAULT_MAX_SERVER_CACHE_SIZE); long estimatedSize = Math.min(scanner.getEstimatedSize(), maxSize); if (estimatedSize > Integer.MAX_VALUE) { throw new IllegalStateException("Estimated size(" + estimatedSize + ") must not be greater than Integer.MAX_VALUE(" + Integer.MAX_VALUE + ")"); diff --git a/src/main/java/com/salesforce/phoenix/query/QueryServices.java b/src/main/java/com/salesforce/phoenix/query/QueryServices.java index 5a106217..c3016bae 100644 --- a/src/main/java/com/salesforce/phoenix/query/QueryServices.java +++ b/src/main/java/com/salesforce/phoenix/query/QueryServices.java @@ -158,7 +158,7 @@ public interface QueryServices extends SQLCloseable { public static final String MAX_MEMORY_PERC_ATTRIB = "phoenix.query.maxGlobalMemoryPercentage"; public static final String MAX_MEMORY_WAIT_MS_ATTRIB = "phoenix.query.maxGlobalMemoryWaitMs"; public static final String MAX_TENANT_MEMORY_PERC_ATTRIB = "phoenix.query.maxTenantMemoryPercentage"; - public static final String MAX_HASH_CACHE_SIZE_ATTRIB = "phoenix.query.maxHashCacheBytes"; + public static final String MAX_SERVER_CACHE_SIZE_ATTRIB = "phoenix.query.maxServerCacheBytes"; public static final String TARGET_QUERY_CONCURRENCY_ATTRIB = "phoenix.query.targetConcurrency"; public static final String MAX_QUERY_CONCURRENCY_ATTRIB = "phoenix.query.maxConcurrency"; public static final String DATE_FORMAT_ATTRIB = "phoenix.query.dateFormat"; diff --git a/src/main/java/com/salesforce/phoenix/query/QueryServicesOptions.java b/src/main/java/com/salesforce/phoenix/query/QueryServicesOptions.java index 2d7aefb2..d3ada876 100644 --- a/src/main/java/com/salesforce/phoenix/query/QueryServicesOptions.java +++ b/src/main/java/com/salesforce/phoenix/query/QueryServicesOptions.java @@ -33,7 +33,7 @@ import static com.salesforce.phoenix.query.QueryServices.IMMUTABLE_ROWS_ATTRIB; import static com.salesforce.phoenix.query.QueryServices.KEEP_ALIVE_MS_ATTRIB; import static com.salesforce.phoenix.query.QueryServices.MASTER_INFO_PORT_ATTRIB; -import static com.salesforce.phoenix.query.QueryServices.MAX_HASH_CACHE_SIZE_ATTRIB; +import static com.salesforce.phoenix.query.QueryServices.MAX_SERVER_CACHE_SIZE_ATTRIB; import static com.salesforce.phoenix.query.QueryServices.MAX_INTRA_REGION_PARALLELIZATION_ATTRIB; import static com.salesforce.phoenix.query.QueryServices.MAX_MEMORY_PERC_ATTRIB; import static com.salesforce.phoenix.query.QueryServices.MAX_MEMORY_WAIT_MS_ATTRIB; @@ -81,7 +81,7 @@ public class QueryServicesOptions { public static final int DEFAULT_MAX_MEMORY_PERC = 50; // 50% of heap public static final int DEFAULT_MAX_MEMORY_WAIT_MS = 10000; public static final int DEFAULT_MAX_TENANT_MEMORY_PERC = 100; - public static final long DEFAULT_MAX_HASH_CACHE_SIZE = 1024*1024*100; // 100 Mb + public static final long DEFAULT_MAX_SERVER_CACHE_SIZE = 1024*1024*100; // 100 Mb public static final int DEFAULT_TARGET_QUERY_CONCURRENCY = 32; public static final int DEFAULT_MAX_QUERY_CONCURRENCY = 64; public static final String DEFAULT_DATE_FORMAT = DateUtil.DEFAULT_DATE_FORMAT; @@ -137,7 +137,7 @@ public static QueryServicesOptions withDefaults() { .setIfUnset(MAX_MEMORY_PERC_ATTRIB, DEFAULT_MAX_MEMORY_PERC) .setIfUnset(MAX_MEMORY_WAIT_MS_ATTRIB, DEFAULT_MAX_MEMORY_WAIT_MS) .setIfUnset(MAX_TENANT_MEMORY_PERC_ATTRIB, DEFAULT_MAX_TENANT_MEMORY_PERC) - .setIfUnset(MAX_HASH_CACHE_SIZE_ATTRIB, DEFAULT_MAX_HASH_CACHE_SIZE) + .setIfUnset(MAX_SERVER_CACHE_SIZE_ATTRIB, DEFAULT_MAX_SERVER_CACHE_SIZE) .setIfUnset(SCAN_CACHE_SIZE_ATTRIB, DEFAULT_SCAN_CACHE_SIZE) .setIfUnset(TARGET_QUERY_CONCURRENCY_ATTRIB, DEFAULT_TARGET_QUERY_CONCURRENCY) .setIfUnset(MAX_QUERY_CONCURRENCY_ATTRIB, DEFAULT_MAX_QUERY_CONCURRENCY) @@ -221,8 +221,8 @@ public QueryServicesOptions setMaxTenantMemoryPerc(int maxTenantMemoryPerc) { return set(MAX_TENANT_MEMORY_PERC_ATTRIB, maxTenantMemoryPerc); } - public QueryServicesOptions setMaxHashCacheSize(long maxHashCacheSize) { - return set(MAX_HASH_CACHE_SIZE_ATTRIB, maxHashCacheSize); + public QueryServicesOptions setMaxServerCacheSize(long maxServerCacheSize) { + return set(MAX_SERVER_CACHE_SIZE_ATTRIB, maxServerCacheSize); } public QueryServicesOptions setScanFetchSize(int scanFetchSize) { diff --git a/src/main/java/com/salesforce/phoenix/schema/KeyValueSchema.java b/src/main/java/com/salesforce/phoenix/schema/KeyValueSchema.java index afd4852a..5014b68f 100644 --- a/src/main/java/com/salesforce/phoenix/schema/KeyValueSchema.java +++ b/src/main/java/com/salesforce/phoenix/schema/KeyValueSchema.java @@ -104,7 +104,7 @@ public byte[] toBytes(Aggregator[] aggregators, ValueBitSet valueSet, ImmutableB // since repeating fields will not span the non-null/null boundary. for (int i = 0; i < fields.size(); i++) { Field field = fields.get(i); - PDataType type = field.getType(); + PDataType type = field.getDataType(); for (int j = 0; j < field.getCount(); j++) { if (aggregators[index].evaluate(null, ptr)) { // Skip null values if (index >= minNullableIndex) { @@ -202,7 +202,7 @@ public Boolean next(ImmutableBytesWritable ptr, int position, int maxOffset, Val ptr.set(ptr.get(), ptr.getOffset() + ptr.getLength(), 0); if (!isNull(position, valueSet)) { Field field = this.getField(position); - if (field.getType().isFixedWidth()) { + if (field.getDataType().isFixedWidth()) { ptr.set(ptr.get(),ptr.getOffset(), field.getByteSize()); } else { int length = ByteUtil.vintFromBytes(ptr); diff --git a/src/main/java/com/salesforce/phoenix/schema/RowKeySchema.java b/src/main/java/com/salesforce/phoenix/schema/RowKeySchema.java index cf6be740..e66a26b3 100644 --- a/src/main/java/com/salesforce/phoenix/schema/RowKeySchema.java +++ b/src/main/java/com/salesforce/phoenix/schema/RowKeySchema.java @@ -141,11 +141,11 @@ public Boolean next(ImmutableBytesWritable ptr, int position, int maxOffset) { // backing byte array. ptr.set(ptr.get(), ptr.getOffset() + ptr.getLength(), 0); // If positioned at SEPARATOR_BYTE, skip it. - if (position > 0 && !getField(position-1).getType().isFixedWidth()) { + if (position > 0 && !getField(position-1).getDataType().isFixedWidth()) { ptr.set(ptr.get(), ptr.getOffset()+ptr.getLength()+1, 0); } Field field = this.getField(position); - if (field.getType().isFixedWidth()) { + if (field.getDataType().isFixedWidth()) { ptr.set(ptr.get(),ptr.getOffset(), field.getByteSize()); } else { if (position+1 == getFieldCount() ) { // Last field has no terminator @@ -170,7 +170,7 @@ public Boolean previous(ImmutableBytesWritable ptr, int position, int minOffset) return null; } Field field = this.getField(position); - if (field.getType().isFixedWidth()) { + if (field.getDataType().isFixedWidth()) { ptr.set(ptr.get(), ptr.getOffset()-field.getByteSize(), field.getByteSize()); return true; } @@ -184,7 +184,7 @@ public Boolean previous(ImmutableBytesWritable ptr, int position, int minOffset) // Field before the one we want to position at is variable length // In this case, we can search backwards for our separator byte // to determine the length - if (!field.getType().isFixedWidth()) { + if (!field.getDataType().isFixedWidth()) { byte[] buf = ptr.get(); int offset = ptr.getOffset()-1; while (offset > minOffset /* sanity check*/ && buf[offset] != QueryConstants.SEPARATOR_BYTE) { @@ -198,7 +198,7 @@ public Boolean previous(ImmutableBytesWritable ptr, int position, int minOffset) return true; } int i,fixedOffset = field.getByteSize(); - for (i = position-2; i >= 0 && this.getField(i).getType().isFixedWidth(); i--) { + for (i = position-2; i >= 0 && this.getField(i).getDataType().isFixedWidth(); i--) { fixedOffset += this.getField(i).getByteSize(); } // All of the previous fields are fixed width, so we can calculate the offset @@ -228,13 +228,13 @@ public Boolean reposition(ImmutableBytesWritable ptr, int oldPosition, int newPo } else { int nVarLengthFromBeginning = 0; for (int i = 0; i <= newPosition; i++) { - if (!this.getField(i).getType().isFixedWidth()) { + if (!this.getField(i).getDataType().isFixedWidth()) { nVarLengthFromBeginning++; } } int nVarLengthBetween = 0; for (int i = oldPosition - 1; i >= newPosition; i--) { - if (!this.getField(i).getType().isFixedWidth()) { + if (!this.getField(i).getDataType().isFixedWidth()) { nVarLengthBetween++; } } diff --git a/src/main/java/com/salesforce/phoenix/schema/ValueSchema.java b/src/main/java/com/salesforce/phoenix/schema/ValueSchema.java index 9bb68836..d340c927 100644 --- a/src/main/java/com/salesforce/phoenix/schema/ValueSchema.java +++ b/src/main/java/com/salesforce/phoenix/schema/ValueSchema.java @@ -72,7 +72,7 @@ private void init(int minNullable, List fields) { int positions = 0; for (Field field : fields) { int fieldEstLength = 0; - PDataType type = field.getType(); + PDataType type = field.getDataType(); Integer byteSize = type.getByteSize(); if (type.isFixedWidth()) { fieldEstLength += field.getByteSize(); @@ -185,7 +185,7 @@ private Field(PDatum datum, boolean isNullable, int count, ColumnModifier column } private Field(Field field, int count) { - this.type = field.getType(); + this.type = field.getDataType(); this.byteSize = field.byteSize; this.count = count; } @@ -194,7 +194,7 @@ public final ColumnModifier getColumnModifier() { return columnModifier; } - public final PDataType getType() { + public final PDataType getDataType() { return type; } diff --git a/src/main/java/com/salesforce/phoenix/util/ScanUtil.java b/src/main/java/com/salesforce/phoenix/util/ScanUtil.java index 9e6f9b57..3d9ae24a 100644 --- a/src/main/java/com/salesforce/phoenix/util/ScanUtil.java +++ b/src/main/java/com/salesforce/phoenix/util/ScanUtil.java @@ -204,7 +204,7 @@ private static byte[] getKey(RowKeySchema schema, List> slots, Bo for (int i = 0; i < position.length; i++) { position[i] = bound == Bound.LOWER ? 0 : slots.get(i).size()-1; KeyRange range = slots.get(i).get(position[i]); - maxLength += range.getRange(bound).length + (schema.getField(i).getType().isFixedWidth() ? 0 : 1); + maxLength += range.getRange(bound).length + (schema.getField(i).getDataType().isFixedWidth() ? 0 : 1); } byte[] key = new byte[maxLength]; int length = setKey(schema, slots, position, bound, key, 0, 0, position.length); @@ -227,7 +227,7 @@ public static int estimateMaximumKeyLength(RowKeySchema schema, int schemaStartI maxLowerRangeLength = Math.max(maxLowerRangeLength, range.getLowerRange().length); maxUpperRangeLength = Math.max(maxUpperRangeLength, range.getUpperRange().length); } - int trailingByte = (schema.getField(schemaStartIndex).getType().isFixedWidth() || + int trailingByte = (schema.getField(schemaStartIndex).getDataType().isFixedWidth() || schemaStartIndex == schema.getFieldCount() - 1 ? 0 : 1); maxLowerKeyLength += maxLowerRangeLength + trailingByte; maxUpperKeyLength += maxUpperKeyLength + trailingByte; @@ -264,7 +264,7 @@ public static int setKey(RowKeySchema schema, List> slots, int[] // Build up the key by appending the bound of each key range // from the current position of each slot. KeyRange range = slots.get(i).get(position[i]); - boolean isFixedWidth = schema.getField(schemaStartIndex++).getType().isFixedWidth(); + boolean isFixedWidth = schema.getField(schemaStartIndex++).getDataType().isFixedWidth(); /* * If the current slot is unbound then stop if: * 1) setting the upper bound. There's no value in @@ -332,7 +332,7 @@ public static int setKey(RowKeySchema schema, List> slots, int[] // byte. if (bound == Bound.LOWER) { while (schemaStartIndex > 0 && offset > byteOffset && - !schema.getField(--schemaStartIndex).getType().isFixedWidth() && + !schema.getField(--schemaStartIndex).getDataType().isFixedWidth() && key[offset-1] == QueryConstants.SEPARATOR_BYTE) { offset--; } diff --git a/src/test/java/com/salesforce/hbase/index/TestEndtoEndIndexingWithCompression.java b/src/test/java/com/salesforce/hbase/index/TestEndtoEndIndexingWithCompression.java index 8cc264b7..87727aaf 100644 --- a/src/test/java/com/salesforce/hbase/index/TestEndtoEndIndexingWithCompression.java +++ b/src/test/java/com/salesforce/hbase/index/TestEndtoEndIndexingWithCompression.java @@ -27,6 +27,7 @@ ******************************************************************************/ package com.salesforce.hbase.index; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec; import org.apache.hadoop.hbase.regionserver.wal.WALEditCodec; @@ -41,9 +42,11 @@ public class TestEndtoEndIndexingWithCompression extends TestEndtoEndIndexing{ @BeforeClass public static void setupCluster() throws Exception { //add our codec and enable WAL compression - UTIL.getConfiguration().set(WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY, - IndexedWALEditCodec.class.getName()); - UTIL.getConfiguration().setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true); + Configuration conf = UTIL.getConfiguration(); + IndexTestingUtils.setupConfig(conf); + conf.set(WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY, + IndexedWALEditCodec.class.getName()); + conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true); //start the mini-cluster UTIL.startMiniCluster(); diff --git a/src/test/java/com/salesforce/phoenix/query/QueryServicesTestImpl.java b/src/test/java/com/salesforce/phoenix/query/QueryServicesTestImpl.java index f484ff5b..477a65b4 100644 --- a/src/test/java/com/salesforce/phoenix/query/QueryServicesTestImpl.java +++ b/src/test/java/com/salesforce/phoenix/query/QueryServicesTestImpl.java @@ -71,7 +71,7 @@ public QueryServicesTestImpl(ReadOnlyProps overrideProps) { .setSpoolThresholdBytes(DEFAULT_SPOOL_THRESHOLD_BYTES) .setMaxMemoryWaitMs(DEFAULT_MAX_MEMORY_WAIT_MS) .setMaxTenantMemoryPerc(DEFAULT_MAX_TENANT_MEMORY_PERC) - .setMaxHashCacheSize(DEFAULT_MAX_HASH_CACHE_SIZE) + .setMaxServerCacheSize(DEFAULT_MAX_HASH_CACHE_SIZE) .setTargetQueryConcurrency(DEFAULT_TARGET_QUERY_CONCURRENCY) .setMaxQueryConcurrency(DEFAULT_MAX_QUERY_CONCURRENCY) .setRowKeyOrderSaltedTable(true) diff --git a/src/test/java/com/salesforce/phoenix/schema/RowKeySchemaTest.java b/src/test/java/com/salesforce/phoenix/schema/RowKeySchemaTest.java index c6066570..e78947b8 100644 --- a/src/test/java/com/salesforce/phoenix/schema/RowKeySchemaTest.java +++ b/src/test/java/com/salesforce/phoenix/schema/RowKeySchemaTest.java @@ -77,7 +77,7 @@ private void assertIteration(String dataColumns, String pk, Object[] values, Str assertTrue(hasValue); PDataType type = PDataType.fromLiteral(values[i]); ColumnModifier mod = mods.get(i); - Object value = type.toObject(ptr, schema.getField(i).getType(), mod); + Object value = type.toObject(ptr, schema.getField(i).getDataType(), mod); assertEquals(values[i], value); } assertEquals(nExpectedValues, i); @@ -91,7 +91,7 @@ private void assertIteration(String dataColumns, String pk, Object[] values, Str assertTrue(hasValue); PDataType type = PDataType.fromLiteral(values[i]); ColumnModifier mod = mods.get(i); - Object value = type.toObject(ptr, schema.getField(i).getType(), mod); + Object value = type.toObject(ptr, schema.getField(i).getDataType(), mod); assertEquals(values[i], value); } assertEquals(-1, i); From 36831e9c72850676b56a22b2aec40a0c602e49f2 Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Fri, 13 Sep 2013 13:34:30 -0700 Subject: [PATCH 021/102] Adding short-circuit skip for indexing codecs. Solves a perf problem where, if the index isn't enabled, it still spends a lot of time breaking up the Put/Delete, just for no work. --- .../covered/CoveredColumnsIndexBuilder.java | 6 ++++ .../hbase/index/covered/IndexCodec.java | 15 ++++++++ .../example/CoveredColumnIndexCodec.java | 9 +++++ .../phoenix/index/PhoenixIndexCodec.java | 36 +++++++++++++++---- 4 files changed, 59 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java b/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java index a0f74fe3..68ebb9d4 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java +++ b/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java @@ -109,6 +109,9 @@ public void setup(RegionCoprocessorEnvironment env) throws IOException { @Override public Collection> getIndexUpdate(Put p) throws IOException { + if (!codec.isEnabled()) { + return null; + } // build the index updates for each group IndexUpdateManager updateMap = new IndexUpdateManager(); @@ -428,6 +431,9 @@ private void cleanupIndexStateFromBatchOnward(IndexUpdateManager updateMap, @Override public Collection> getIndexUpdate(Delete d) throws IOException { + if (!codec.isEnabled()) { + return null; + } // stores all the return values IndexUpdateManager updateMap = new IndexUpdateManager(); diff --git a/src/main/java/com/salesforce/hbase/index/covered/IndexCodec.java b/src/main/java/com/salesforce/hbase/index/covered/IndexCodec.java index d70cc5fb..7a1c84c4 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/IndexCodec.java +++ b/src/main/java/com/salesforce/hbase/index/covered/IndexCodec.java @@ -29,6 +29,7 @@ import java.io.IOException; +import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; @@ -84,4 +85,18 @@ public interface IndexCodec { * @throws IOException */ public Iterable getIndexUpserts(TableState state) throws IOException; + + /** + * This allows the codec to dynamically change whether or not indexing should take place for a + * table. If it doesn't take place, we can save a lot of time on the regular Put patch. By making + * it dynamic, we can save offlining and then onlining a table just to turn indexing on. + *

+ * We can also be smart about even indexing a given update here too - if the update doesn't + * contain any columns that we care about indexing, we can save the effort of analyzing the put + * and further. + * @param m mutation that should be indexed. + * @return true if indexing is enabled for the given table. This should be on a per-table + * basis, as each codec is instantiated per-region. + */ + public boolean isEnabled(Mutation m); } \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexCodec.java b/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexCodec.java index 61f1a789..c9e51919 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexCodec.java +++ b/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexCodec.java @@ -20,12 +20,14 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Iterator; import java.util.List; import java.util.Map.Entry; import org.apache.commons.lang.ArrayUtils; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.util.Bytes; @@ -352,4 +354,11 @@ public static boolean checkRowKeyForAllNulls(byte[] bytes) { return true; } + + @Override + public boolean isEnabled(Mutation m) { + // this could be a bit smarter, looking at the groups for the mutation, but we leave it at this + // simple check for the moment. + return groups.size() > 0; + } } \ No newline at end of file diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java index bbb3062b..19200535 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java @@ -18,8 +18,11 @@ import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; @@ -51,29 +54,43 @@ public class PhoenixIndexCodec implements IndexCodec { private List indexMaintainers; private final ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + private Configuration conf; @Override public void initialize(RegionCoprocessorEnvironment env) { + this.conf = env.getConfiguration(); } private List getIndexMaintainers(TableState state) { + return getIndexMaintainers(state.getUpdateAttributes()); + } + + /** + * @param attributes attributes from a primary table update + * @return the {@link IndexMaintainer}s that would maintain the index for an update with the + * attributes. + */ + private List getIndexMaintainers(Map attributes) { if (indexMaintainers != null) { return indexMaintainers; } + byte[] md; - byte[] uuid = state.getUpdateAttributes().get(INDEX_UUID); + byte[] uuid = attributes.get(INDEX_UUID); if (uuid == null) { - md = state.getUpdateAttributes().get(INDEX_MD); + md = attributes.get(INDEX_MD); if (md == null) { indexMaintainers = Collections.emptyList(); } else { indexMaintainers = IndexMaintainer.deserialize(md); } } else { - byte[] tenantIdBytes = state.getUpdateAttributes().get(PhoenixRuntime.TENANT_ID_ATTRIB); - ImmutableBytesWritable tenantId = tenantIdBytes == null ? null : new ImmutableBytesWritable(tenantIdBytes); - TenantCache cache = GlobalCache.getTenantCache(state.getEnvironment().getConfiguration(), tenantId); - IndexMetaDataCache indexCache = (IndexMetaDataCache)cache.getServerCache(new ImmutableBytesPtr(uuid)); + byte[] tenantIdBytes = attributes.get(PhoenixRuntime.TENANT_ID_ATTRIB); + ImmutableBytesWritable tenantId = + tenantIdBytes == null ? null : new ImmutableBytesWritable(tenantIdBytes); + TenantCache cache = GlobalCache.getTenantCache(conf, tenantId); + IndexMetaDataCache indexCache = + (IndexMetaDataCache) cache.getServerCache(new ImmutableBytesPtr(uuid)); this.indexMaintainers = indexCache.getIndexMaintainers(); } return indexMaintainers; @@ -141,4 +158,9 @@ public Iterable getIndexDeletes(TableState state) throws IOExceptio return indexUpdates; } -} + @Override + public boolean isEnabled(Mutation m) { + List maintainers = getIndexMaintainers(m.getAttributesMap()); + return maintainers.size() > 0; + } +} \ No newline at end of file From 95281acb22b4a4606c6cf530f5d8c93f11995b0b Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Fri, 13 Sep 2013 13:59:06 -0700 Subject: [PATCH 022/102] Adding a code formatter. Set the template via Right-Click on project ->Properties -> Java Code Style -> Formatter --- dev/PhoenixCodeTemplate.xml | 291 ++++++++++++++++++++++++++++++++++++ 1 file changed, 291 insertions(+) create mode 100644 dev/PhoenixCodeTemplate.xml diff --git a/dev/PhoenixCodeTemplate.xml b/dev/PhoenixCodeTemplate.xml new file mode 100644 index 00000000..4bae1d34 --- /dev/null +++ b/dev/PhoenixCodeTemplate.xml @@ -0,0 +1,291 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 702810cce0d1e49ccbc07642340b08108b8074ab Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Fri, 13 Sep 2013 14:05:12 -0700 Subject: [PATCH 023/102] Slight fix for short-circuit indexing --- .../hbase/index/covered/CoveredColumnsIndexBuilder.java | 4 ++-- .../hbase/index/covered/CoveredIndexCodecForTesting.java | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java b/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java index 68ebb9d4..238aef97 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java +++ b/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java @@ -109,7 +109,7 @@ public void setup(RegionCoprocessorEnvironment env) throws IOException { @Override public Collection> getIndexUpdate(Put p) throws IOException { - if (!codec.isEnabled()) { + if (!codec.isEnabled(p)) { return null; } // build the index updates for each group @@ -431,7 +431,7 @@ private void cleanupIndexStateFromBatchOnward(IndexUpdateManager updateMap, @Override public Collection> getIndexUpdate(Delete d) throws IOException { - if (!codec.isEnabled()) { + if (!codec.isEnabled(d)) { return null; } // stores all the return values diff --git a/src/test/java/com/salesforce/hbase/index/covered/CoveredIndexCodecForTesting.java b/src/test/java/com/salesforce/hbase/index/covered/CoveredIndexCodecForTesting.java index c6cac611..3da7fa6d 100644 --- a/src/test/java/com/salesforce/hbase/index/covered/CoveredIndexCodecForTesting.java +++ b/src/test/java/com/salesforce/hbase/index/covered/CoveredIndexCodecForTesting.java @@ -32,6 +32,7 @@ import java.util.Arrays; import java.util.List; +import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import com.salesforce.hbase.index.covered.IndexCodec; @@ -72,4 +73,9 @@ public Iterable getIndexUpserts(TableState state) { public void initialize(RegionCoprocessorEnvironment env) throws IOException { // noop } + + @Override + public boolean isEnabled(Mutation m) { + return true; + } } \ No newline at end of file From 5a7c35b33d486702fd346a1ef99a4e0467e7f545 Mon Sep 17 00:00:00 2001 From: Matt Walsh Date: Fri, 13 Sep 2013 16:33:03 -0500 Subject: [PATCH 024/102] Changes to make hashing more efficient. --- .../expression/function/MD5Function.java | 107 +++++++++++------- .../phoenix/end2end/MD5FunctionTest.java | 61 ++++++++-- 2 files changed, 121 insertions(+), 47 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/expression/function/MD5Function.java b/src/main/java/com/salesforce/phoenix/expression/function/MD5Function.java index 2b1e80bc..8e8a7152 100644 --- a/src/main/java/com/salesforce/phoenix/expression/function/MD5Function.java +++ b/src/main/java/com/salesforce/phoenix/expression/function/MD5Function.java @@ -1,3 +1,30 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ package com.salesforce.phoenix.expression.function; import java.security.MessageDigest; @@ -8,58 +35,60 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import com.salesforce.phoenix.expression.Expression; -import com.salesforce.phoenix.parse.FunctionParseNode; +import com.salesforce.phoenix.parse.FunctionParseNode.*; import com.salesforce.phoenix.schema.PDataType; import com.salesforce.phoenix.schema.tuple.Tuple; -@FunctionParseNode.BuiltInFunction(name=MD5Function.NAME, args={ - @FunctionParseNode.Argument(allowedTypes={PDataType.VARCHAR})} ) +@BuiltInFunction(name = MD5Function.NAME, args={@Argument()}) public class MD5Function extends ScalarFunction { - public static final String NAME = "MD5"; + public static final String NAME = "MD5"; - public MD5Function() { - } + private final MessageDigest messageDigest; - public MD5Function(List children) throws SQLException { - super(children); + public MD5Function(List children) throws SQLException { + super(children); + try { + messageDigest = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); } + } - @Override - public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { - if (!getStrExpression().evaluate(tuple, ptr)) { - return false; - } - - String sourceStr = (String)PDataType.VARCHAR.toObject(ptr, getStrExpression().getColumnModifier()); + @Override + public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { + if (!getChildExpression().evaluate(tuple, ptr)) { + return false; + } - if (sourceStr == null) { - return true; - } + String sourceStr = (String) PDataType.VARCHAR.toObject(ptr, getChildExpression().getColumnModifier()); - try { - ptr.set(MessageDigest.getInstance("MD5").digest(sourceStr.getBytes())); - return true; - } catch (NoSuchAlgorithmException e) { - throw new IllegalArgumentException(e); - } + if (sourceStr == null) { + return true; } - @Override - public PDataType getDataType() { - return getStrExpression().getDataType(); - } + // Update the digest value + messageDigest.update(ptr.get(), ptr.getOffset(), ptr.getLength()); + // Get the digest bytes (note this resets the messageDigest as well) + ptr.set(messageDigest.digest()); + return true; + } - @Override - public boolean isNullable() { - return getStrExpression().isNullable(); - } + @Override + public PDataType getDataType() { + return PDataType.BINARY; + } - @Override - public String getName() { - return NAME; - } + @Override + public boolean isNullable() { + return getChildExpression().isNullable(); + } - private Expression getStrExpression() { - return children.get(0); - } + @Override + public String getName() { + return NAME; + } + + private Expression getChildExpression() { + return children.get(0); + } } diff --git a/src/test/java/com/salesforce/phoenix/end2end/MD5FunctionTest.java b/src/test/java/com/salesforce/phoenix/end2end/MD5FunctionTest.java index 923f86c4..ae6b23d4 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/MD5FunctionTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/MD5FunctionTest.java @@ -1,3 +1,30 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ package com.salesforce.phoenix.end2end; import static org.junit.Assert.*; @@ -12,22 +39,40 @@ public class MD5FunctionTest extends BaseHBaseManagedTimeTest { @Test - public void testReverse() throws Exception { + public void testRetrieve() throws Exception { String testString = "mwalsh"; Connection conn = DriverManager.getConnection(getUrl()); - String ddl = "CREATE TABLE IF NOT EXISTS MD5_TEST (pk VARCHAR NOT NULL PRIMARY KEY)"; + String ddl = "CREATE TABLE IF NOT EXISTS MD5_RETRIEVE_TEST (pk VARCHAR NOT NULL PRIMARY KEY)"; conn.createStatement().execute(ddl); - String dml = String.format("UPSERT INTO MD5_TEST VALUES('%s')", testString); + String dml = String.format("UPSERT INTO MD5_RETRIEVE_TEST VALUES('%s')", testString); conn.createStatement().execute(dml); conn.commit(); - ResultSet rs; - rs = conn.createStatement().executeQuery("SELECT MD5(pk) FROM MD5_TEST"); + ResultSet rs = conn.createStatement().executeQuery("SELECT MD5(pk) FROM MD5_RETRIEVE_TEST"); assertTrue(rs.next()); - String first = new String(MessageDigest.getInstance("MD5").digest(testString.getBytes())); - String second = rs.getString(1); - assertEquals(first, second); + byte[] first = MessageDigest.getInstance("MD5").digest(testString.getBytes()); + byte[] second = rs.getBytes(1); + assertArrayEquals(first, second); + assertFalse(rs.next()); + } + + @Test + public void testUpsert() throws Exception { + String testString = "mwalsh"; + + Connection conn = DriverManager.getConnection(getUrl()); + String ddl = "CREATE TABLE IF NOT EXISTS MD5_UPSERT_TEST (pk binary(16) NOT NULL PRIMARY KEY)"; + conn.createStatement().execute(ddl); + String dml = String.format("UPSERT INTO MD5_UPSERT_TEST VALUES(md5('%s'))", testString); + conn.createStatement().execute(dml); + conn.commit(); + + ResultSet rs = conn.createStatement().executeQuery("SELECT pk FROM MD5_UPSERT_TEST"); + assertTrue(rs.next()); + byte[] first = MessageDigest.getInstance("MD5").digest(testString.getBytes()); + byte[] second = rs.getBytes(1); + assertArrayEquals(first, second); assertFalse(rs.next()); } From 1191b697f5523cdf257c132d76b9d492d7dc3202 Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Fri, 13 Sep 2013 18:36:06 -0700 Subject: [PATCH 025/102] Adding PhoenixIndexBuilder to handle full short-circuting. Previously, short circuiting only work on the building the index side. However, that meant that for every mutation we were still checking each KV to see if it was an index type. This lead to a 20% slowdown from the case where indexing codec wasn't even installed. There is a little bit of work to be done here as we aren't completely smart about managing the IndexMaintainers - we re-find the set of maintainers for each Mutation everytime we need them. Its a little bit of an overhead, but not terrible. It can be fixed later if that's still a sticking point. --- .../com/salesforce/hbase/index/Indexer.java | 20 +++++++-- .../hbase/index/builder/BaseIndexBuilder.java | 10 +++++ .../hbase/index/builder/IndexBuilder.java | 14 ++++++ .../covered/CoveredColumnsIndexBuilder.java | 8 +--- .../phoenix/index/PhoenixIndexBuilder.java | 44 +++++++++++++++++++ .../phoenix/index/PhoenixIndexCodec.java | 6 ++- .../query/ConnectionQueryServicesImpl.java | 5 ++- .../salesforce/phoenix/util/CSVLoader.java | 4 +- 8 files changed, 96 insertions(+), 15 deletions(-) create mode 100644 src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java diff --git a/src/main/java/com/salesforce/hbase/index/Indexer.java b/src/main/java/com/salesforce/hbase/index/Indexer.java index 0e67ebb9..53b8360e 100644 --- a/src/main/java/com/salesforce/hbase/index/Indexer.java +++ b/src/main/java/com/salesforce/hbase/index/Indexer.java @@ -150,6 +150,9 @@ public void start(CoprocessorEnvironment e) throws IOException { public void prePut(final ObserverContext c, final Put put, final WALEdit edit, final boolean writeToWAL) throws IOException { // get the mapping for index column -> target index table + if (!this.builder.isEnabled(put)) { + return; + } Collection> indexUpdates = this.builder.getIndexUpdate(put); doPre(indexUpdates, edit, writeToWAL); @@ -158,6 +161,10 @@ public void prePut(final ObserverContext c, final @Override public void preDelete(ObserverContext e, Delete delete, WALEdit edit, boolean writeToWAL) throws IOException { + // short circuit so we don't waste time + if (!this.builder.isEnabled(delete)) { + return; + } // get the mapping for index column -> target index table Collection> indexUpdates = this.builder.getIndexUpdate(delete); @@ -207,25 +214,32 @@ public void postBatchMutate(ObserverContext c, @Override public void postPut(ObserverContext e, Put put, WALEdit edit, boolean writeToWAL) throws IOException { - doPost(edit, writeToWAL); + doPost(edit, put, writeToWAL); } @Override public void postDelete(ObserverContext e, Delete delete, WALEdit edit, boolean writeToWAL) throws IOException { - doPost(edit, writeToWAL); + doPost(edit,delete, writeToWAL); } /** * @param edit * @param writeToWAL */ - private void doPost(WALEdit edit, boolean writeToWAL) { + private void doPost(WALEdit edit, Mutation m, boolean writeToWAL) { if (!writeToWAL) { // already did the index update in prePut, so we are done return; } + // turns out, even doing the checking here for the index updates will cause a huge slowdown. Its + // up to the codec to be smart about how it manages this (and leak a little of the + // implementation here, but that's the way optimization go). + if(!this.builder.isEnabled(m)){ + return; + } + Collection> indexUpdates = extractIndexUpdate(edit); // early exit - we have nothing to write, so we don't need to do anything else. NOTE: we don't diff --git a/src/main/java/com/salesforce/hbase/index/builder/BaseIndexBuilder.java b/src/main/java/com/salesforce/hbase/index/builder/BaseIndexBuilder.java index 5c20d7ff..07181650 100644 --- a/src/main/java/com/salesforce/hbase/index/builder/BaseIndexBuilder.java +++ b/src/main/java/com/salesforce/hbase/index/builder/BaseIndexBuilder.java @@ -65,4 +65,14 @@ public void batchStarted(MiniBatchOperationInProgress> m public void batchCompleted(MiniBatchOperationInProgress> miniBatchOp) { // noop } + + /** + * By default, we always attempt to index the mutation. Commonly this can be slow (because the + * framework spends the time to do the indexing, only to realize that you don't need it) or not + * ideal (if you want to turn on/off indexing on a table without completely reloading it). + */ + @Override + public boolean isEnabled(Mutation m) { + return true; + } } \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/builder/IndexBuilder.java b/src/main/java/com/salesforce/hbase/index/builder/IndexBuilder.java index e3fc48b8..ae98da92 100644 --- a/src/main/java/com/salesforce/hbase/index/builder/IndexBuilder.java +++ b/src/main/java/com/salesforce/hbase/index/builder/IndexBuilder.java @@ -108,4 +108,18 @@ public Collection> getIndexUpdateForFilteredRows( * @param miniBatchOp the full batch operation to be written */ public void batchStarted(MiniBatchOperationInProgress> miniBatchOp); + + /** + * This allows the codec to dynamically change whether or not indexing should take place for a + * table. If it doesn't take place, we can save a lot of time on the regular Put patch. By making + * it dynamic, we can save offlining and then onlining a table just to turn indexing on. + *

+ * We can also be smart about even indexing a given update here too - if the update doesn't + * contain any columns that we care about indexing, we can save the effort of analyzing the put + * and further. + * @param m mutation that should be indexed. + * @return true if indexing is enabled for the given table. This should be on a per-table + * basis, as each codec is instantiated per-region. + */ + public boolean isEnabled(Mutation m); } \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java b/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java index 238aef97..075dbab1 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java +++ b/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java @@ -82,7 +82,7 @@ public class CoveredColumnsIndexBuilder extends BaseIndexBuilder { public static final String CODEC_CLASS_NAME_KEY = "com.salesforce.hbase.index.codec.class"; protected RegionCoprocessorEnvironment env; - private IndexCodec codec; + protected IndexCodec codec; protected LocalHBaseState localTable; @Override @@ -109,9 +109,6 @@ public void setup(RegionCoprocessorEnvironment env) throws IOException { @Override public Collection> getIndexUpdate(Put p) throws IOException { - if (!codec.isEnabled(p)) { - return null; - } // build the index updates for each group IndexUpdateManager updateMap = new IndexUpdateManager(); @@ -431,9 +428,6 @@ private void cleanupIndexStateFromBatchOnward(IndexUpdateManager updateMap, @Override public Collection> getIndexUpdate(Delete d) throws IOException { - if (!codec.isEnabled(d)) { - return null; - } // stores all the return values IndexUpdateManager updateMap = new IndexUpdateManager(); diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java new file mode 100644 index 00000000..c0adcdc2 --- /dev/null +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java @@ -0,0 +1,44 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.phoenix.index; + +import org.apache.hadoop.hbase.client.Mutation; + +import com.salesforce.hbase.index.covered.CoveredColumnsIndexBuilder; + +/** + * Index builder for covered-columns index that ties into phoenix for faster use. + */ +public class PhoenixIndexBuilder extends CoveredColumnsIndexBuilder { + + @Override + public boolean isEnabled(Mutation m) { + // ask the codec to see if we should even attempt indexing + return this.codec.isEnabled(m); + } +} \ No newline at end of file diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java index 19200535..96829d6a 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java @@ -160,7 +160,9 @@ public Iterable getIndexDeletes(TableState state) throws IOExceptio @Override public boolean isEnabled(Mutation m) { - List maintainers = getIndexMaintainers(m.getAttributesMap()); - return maintainers.size() > 0; + // TODO cache these maintainers so we don't need to rediscover them later (e.g. when building + // the index update) + List maintainers = getIndexMaintainers(m.getAttributesMap()); + return maintainers.size() > 0; } } \ No newline at end of file diff --git a/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java b/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java index ed5e48c8..ce6b35e2 100644 --- a/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java +++ b/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java @@ -104,6 +104,7 @@ import com.salesforce.phoenix.exception.SQLExceptionCode; import com.salesforce.phoenix.exception.SQLExceptionInfo; import com.salesforce.phoenix.execute.MutationState; +import com.salesforce.phoenix.index.PhoenixIndexBuilder; import com.salesforce.phoenix.index.PhoenixIndexCodec; import com.salesforce.phoenix.jdbc.PhoenixConnection; import com.salesforce.phoenix.jdbc.PhoenixDatabaseMetaData; @@ -517,7 +518,7 @@ private HTableDescriptor generateTableDescriptor(byte[] tableName, HTableDescrip if (tableType != PTableType.INDEX && !descriptor.hasCoprocessor(Indexer.class.getName())) { Map opts = Maps.newHashMapWithExpectedSize(1); opts.put(CoveredColumnsIndexBuilder.CODEC_CLASS_NAME_KEY, PhoenixIndexCodec.class.getName()); - Indexer.enableIndexing(descriptor, CoveredColumnsIndexBuilder.class, opts); + Indexer.enableIndexing(descriptor, PhoenixIndexBuilder.class, opts); } // Setup split policy on Phoenix metadata table to ensure that the key values of a Phoenix table @@ -834,7 +835,7 @@ private void upgradeTablesFrom2_0to2_1(HBaseAdmin admin, HTableDescriptor newDes } else if (!existingDesc.hasCoprocessor(Indexer.class.getName())) { Map opts = Maps.newHashMapWithExpectedSize(1); opts.put(CoveredColumnsIndexBuilder.CODEC_CLASS_NAME_KEY, PhoenixIndexCodec.class.getName()); - Indexer.enableIndexing(existingDesc, CoveredColumnsIndexBuilder.class, opts); + Indexer.enableIndexing(existingDesc, PhoenixIndexBuilder.class, opts); wasModified = true; } if (wasModified) { diff --git a/src/main/java/com/salesforce/phoenix/util/CSVLoader.java b/src/main/java/com/salesforce/phoenix/util/CSVLoader.java index 8b3519e2..ee88ce77 100644 --- a/src/main/java/com/salesforce/phoenix/util/CSVLoader.java +++ b/src/main/java/com/salesforce/phoenix/util/CSVLoader.java @@ -121,7 +121,9 @@ public void upsert(CSVReader reader) throws Exception { if (columnInfo[index] == null) { continue; } - upsertValue = convertTypeSpecificValue(nextLine[index], columnInfo[index].getSqlType()); + String line = nextLine[index]; + Integer info = columnInfo[index].getSqlType(); + upsertValue = convertTypeSpecificValue(line, info); if (upsertValue != null) { stmt.setObject(index + 1, upsertValue, columnInfo[index].getSqlType()); } else { From 3c3f0effa01d5cc71c95448bf20683ba59b87592 Mon Sep 17 00:00:00 2001 From: maryannxue Date: Sat, 14 Sep 2013 00:08:10 -0400 Subject: [PATCH 026/102] Compiler code complete for hashjoin --- .../phoenix/compile/ColumnProjector.java | 1 + .../phoenix/compile/ColumnResolver.java | 6 + .../phoenix/compile/ExpressionCompiler.java | 12 +- .../phoenix/compile/FromCompiler.java | 20 +- .../phoenix/compile/JoinCompiler.java | 483 +++++++++++++++--- .../phoenix/compile/PostDDLCompiler.java | 5 + .../phoenix/compile/ProjectionCompiler.java | 29 +- .../phoenix/compile/QueryCompiler.java | 52 +- .../phoenix/compile/StatementContext.java | 20 +- .../phoenix/compile/WhereCompiler.java | 35 +- .../GroupedAggregateRegionObserver.java | 1 + .../coprocessor/HashJoinRegionScanner.java | 1 + .../coprocessor/ScanRegionObserver.java | 3 +- .../UngroupedAggregateRegionObserver.java | 1 + .../expression/KeyValueColumnExpression.java | 7 + .../expression/RowKeyColumnExpression.java | 42 +- .../filter/MultiKeyValueComparisonFilter.java | 36 +- .../filter/RowKeyComparisonFilter.java | 5 + .../{coprocessor => join}/ScanProjector.java | 33 +- .../salesforce/phoenix/schema/ColumnRef.java | 23 +- .../schema/tuple/MultiKeyValueTuple.java | 14 + .../phoenix/schema/tuple/ResultTuple.java | 14 + .../schema/tuple/SingleKeyValueTuple.java | 11 + .../phoenix/schema/tuple/Tuple.java | 12 + 24 files changed, 737 insertions(+), 129 deletions(-) rename src/main/java/com/salesforce/phoenix/{coprocessor => join}/ScanProjector.java (89%) diff --git a/src/main/java/com/salesforce/phoenix/compile/ColumnProjector.java b/src/main/java/com/salesforce/phoenix/compile/ColumnProjector.java index 4e2cba53..0c155fb6 100644 --- a/src/main/java/com/salesforce/phoenix/compile/ColumnProjector.java +++ b/src/main/java/com/salesforce/phoenix/compile/ColumnProjector.java @@ -57,6 +57,7 @@ public interface ColumnProjector { */ public Expression getExpression(); + // TODO: An expression may contain references to multiple tables. /** * Get the name of the hbase table containing the column * @return the hbase table name diff --git a/src/main/java/com/salesforce/phoenix/compile/ColumnResolver.java b/src/main/java/com/salesforce/phoenix/compile/ColumnResolver.java index 8db583f2..b45528d9 100644 --- a/src/main/java/com/salesforce/phoenix/compile/ColumnResolver.java +++ b/src/main/java/com/salesforce/phoenix/compile/ColumnResolver.java @@ -59,4 +59,10 @@ public interface ColumnResolver { * @throws AmbiguousColumnException if the column name is ambiguous */ public ColumnRef resolveColumn(String schemaName, String tableName, String colName) throws SQLException; + + /** + * Set if disambiguateWithTable when resolving a column. + * @param disambiguateWithTable + */ + public void setDisambiguateWithTable(boolean disambiguateWithTable); } diff --git a/src/main/java/com/salesforce/phoenix/compile/ExpressionCompiler.java b/src/main/java/com/salesforce/phoenix/compile/ExpressionCompiler.java index 1f49eaf3..2b3a8576 100644 --- a/src/main/java/com/salesforce/phoenix/compile/ExpressionCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/ExpressionCompiler.java @@ -53,6 +53,7 @@ public class ExpressionCompiler extends UnsupportedAllParseNodeVisitor tables; ExpressionCompiler(StatementContext context) { this(context,GroupBy.EMPTY_GROUP_BY); @@ -61,6 +62,7 @@ public class ExpressionCompiler extends UnsupportedAllParseNodeVisitor(1); } public boolean isAggregate() { @@ -71,9 +73,14 @@ public boolean isTopLevel() { return nodeCount == 0; } + public List getTables() { + return tables; + } + public void reset() { this.isAggregate = false; this.nodeCount = 0; + this.tables.clear(); } @Override @@ -367,7 +374,10 @@ protected ColumnRef resolveColumn(ColumnParseNode node) throws SQLException { @Override public Expression visit(ColumnParseNode node) throws SQLException { ColumnRef ref = resolveColumn(node); - if (!SchemaUtil.isPKColumn(ref.getColumn())) { // project only kv columns + TableRef tableRef = ref.getTableRef(); + tables.add(tableRef.getTable()); + if (tableRef.equals(context.getCurrentTable()) + && !SchemaUtil.isPKColumn(ref.getColumn())) { // project only kv columns context.getScan().addColumn(ref.getColumn().getFamilyName().getBytes(), ref.getColumn().getName().getBytes()); } Expression expression = ref.newColumnExpression(); diff --git a/src/main/java/com/salesforce/phoenix/compile/FromCompiler.java b/src/main/java/com/salesforce/phoenix/compile/FromCompiler.java index 022b0c57..d62080c8 100644 --- a/src/main/java/com/salesforce/phoenix/compile/FromCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/FromCompiler.java @@ -46,6 +46,10 @@ public List getTables() { public ColumnRef resolveColumn(String schemaName, String tableName, String colName) throws SQLException { throw new UnsupportedOperationException(); } + + @Override + public void setDisambiguateWithTable(boolean disambiguateWithTable) { + } }; public static ColumnResolver getResolver(final CreateTableStatement statement, final PhoenixConnection connection) @@ -69,7 +73,6 @@ public static ColumnResolver getResolver(final CreateTableStatement statement, f public static ColumnResolver getResolver(SelectStatement statement, PhoenixConnection connection) throws SQLException { List fromNodes = statement.getFrom(); - if (fromNodes.size() > 1) { throw new SQLFeatureNotSupportedException("Joins not supported"); } MultiTableColumnResolver visitor = new MultiTableColumnResolver(connection); for (TableNode node : fromNodes) { node.accept(visitor); @@ -147,10 +150,17 @@ public ColumnRef resolveColumn(String schemaName, String tableName, private static abstract class BaseColumnResolver implements ColumnResolver { protected final PhoenixConnection connection; protected final MetaDataClient client; + protected boolean disambiguateWithTable; private BaseColumnResolver(PhoenixConnection connection) { this.connection = connection; this.client = new MetaDataClient(connection); + this.disambiguateWithTable = false; + } + + @Override + public void setDisambiguateWithTable(boolean disambiguateWithTable) { + this.disambiguateWithTable = disambiguateWithTable; } protected PTable addDynamicColumns(List dynColumns, PTable theTable) @@ -200,7 +210,7 @@ public void visit(BindTableNode boundTableNode) throws SQLException { @Override public void visit(JoinTableNode joinNode) throws SQLException { - throw new SQLFeatureNotSupportedException(); + joinNode.getTable().accept(this); } @SuppressWarnings("serial") @@ -324,18 +334,18 @@ public ColumnRef resolveColumn(String schemaName, String tableName, String colNa } } - if (theTableRef != null) { return new ColumnRef(theTableRef, theColumnPosition); } + if (theTableRef != null) { return new ColumnRef(theTableRef, theColumnPosition, disambiguateWithTable); } throw new ColumnNotFoundException(colName); } else { try { TableRef tableRef = resolveTable(schemaName, tableName); PColumn column = tableRef.getTable().getColumn(colName); - return new ColumnRef(tableRef, column.getPosition()); + return new ColumnRef(tableRef, column.getPosition(), disambiguateWithTable); } catch (TableNotFoundException e) { // Try using the tableName as a columnFamily reference instead ColumnFamilyRef cfRef = resolveColumnFamily(schemaName, tableName); PColumn column = cfRef.getFamily().getColumn(colName); - return new ColumnRef(cfRef.getTableRef(), column.getPosition()); + return new ColumnRef(cfRef.getTableRef(), column.getPosition(), disambiguateWithTable); } } } diff --git a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java index 4eb90654..7e82bd71 100644 --- a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java @@ -28,24 +28,46 @@ package com.salesforce.phoenix.compile; import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.Bytes; -import com.salesforce.phoenix.coprocessor.ScanProjector; -import com.salesforce.phoenix.coprocessor.ScanProjector.ProjectionType; import com.salesforce.phoenix.expression.AndExpression; import com.salesforce.phoenix.expression.Expression; -import com.salesforce.phoenix.jdbc.PhoenixConnection; +import com.salesforce.phoenix.join.ScanProjector; +import com.salesforce.phoenix.join.ScanProjector.ProjectionType; import com.salesforce.phoenix.parse.AliasedNode; +import com.salesforce.phoenix.parse.AndParseNode; +import com.salesforce.phoenix.parse.CaseParseNode; +import com.salesforce.phoenix.parse.ColumnParseNode; +import com.salesforce.phoenix.parse.ComparisonParseNode; +import com.salesforce.phoenix.parse.ConcreteTableNode; +import com.salesforce.phoenix.parse.EqualParseNode; +import com.salesforce.phoenix.parse.InListParseNode; +import com.salesforce.phoenix.parse.IsNullParseNode; +import com.salesforce.phoenix.parse.JoinTableNode; +import com.salesforce.phoenix.parse.LikeParseNode; +import com.salesforce.phoenix.parse.NotParseNode; +import com.salesforce.phoenix.parse.OrParseNode; +import com.salesforce.phoenix.parse.OrderByNode; import com.salesforce.phoenix.parse.ParseNode; import com.salesforce.phoenix.parse.ParseNodeFactory; import com.salesforce.phoenix.parse.SelectStatement; +import com.salesforce.phoenix.parse.StatelessTraverseAllParseNodeVisitor; import com.salesforce.phoenix.parse.TableNode; +import com.salesforce.phoenix.parse.TraverseNoParseNodeVisitor; +import com.salesforce.phoenix.parse.WildcardParseNode; import com.salesforce.phoenix.parse.JoinTableNode.JoinType; +import com.salesforce.phoenix.schema.ColumnRef; import com.salesforce.phoenix.schema.TableRef; -import com.salesforce.phoenix.util.ByteUtil; import com.salesforce.phoenix.util.SchemaUtil; @@ -63,19 +85,51 @@ public static class JoinSpec { private List preFilters; private List postFilters; private List joinTables; - private boolean isPostAggregateOrDistinct; - private ColumnResolver resolver; + private Set columnRefs; + + private JoinSpec(SelectStatement statement, ColumnResolver resolver) throws SQLException { + List selectList = statement.getSelect(); + List tableNodes = statement.getFrom(); + assert (tableNodes.size() > 1); + Iterator iter = tableNodes.iterator(); + Iterator tableRefIter = resolver.getTables().iterator(); + iter.next(); + this.mainTable = tableRefIter.next(); + this.select = extractFromSelect(selectList, mainTable, resolver); + this.joinTables = new ArrayList(tableNodes.size() - 1); + ColumnParseNodeVisitor visitor = new ColumnParseNodeVisitor(resolver); + TableNode tableNode = null; + for (; iter.hasNext(); tableNode = iter.next()) { + if (!(tableNode instanceof JoinTableNode)) + throw new SQLFeatureNotSupportedException("Full joins not supported."); + JoinTableNode joinTableNode = (JoinTableNode) tableNode; + JoinTable joinTable = new JoinTable(joinTableNode, tableRefIter.next(), selectList, resolver); + joinTables.add(joinTable); + joinTableNode.getOnNode().accept(visitor); + } + statement.getWhere().accept(new WhereNodeVisitor(resolver)); + for (AliasedNode node : selectList) { + node.getNode().accept(visitor); + } + statement.getWhere().accept(visitor); + for (ParseNode node : statement.getGroupBy()) { + node.accept(visitor); + } + statement.getHaving().accept(visitor); + for (OrderByNode node : statement.getOrderBy()) { + node.getNode().accept(visitor); + } + this.columnRefs = visitor.getColumnRefMap().keySet(); + } private JoinSpec(TableRef table, List select, List preFilters, - List postFilters, List joinTables, boolean isPostAggregate, - ColumnResolver resolver) { + List postFilters, List joinTables, Set columnRefs) { this.mainTable = table; this.select = select; this.preFilters = preFilters; this.postFilters = postFilters; this.joinTables = joinTables; - this.isPostAggregateOrDistinct = isPostAggregate; - this.resolver = resolver; + this.columnRefs = columnRefs; } public TableRef getMainTable() { @@ -98,14 +152,6 @@ public List getJoinTables() { return joinTables; } - public boolean isPostAggregateOrDistinct() { - return isPostAggregateOrDistinct; - } - - public ColumnResolver getColumnResolver() { - return resolver; - } - public ParseNode getPreFiltersCombined() { if (preFilters == null || preFilters.isEmpty()) return null; @@ -123,18 +169,112 @@ public ScanProjector getScanProjector() { } else { tableName = mainTable.getTableName(); } - byte[] tablePrefix = ByteUtil.concat(tableName, PROJECTION_SEPERATOR); - return new ScanProjector(ProjectionType.TABLE, tablePrefix, null, null); + return new ScanProjector(ProjectionType.TABLE, tableName, null, null); + } + + public List compilePostFilterExpressions(StatementContext context) throws SQLException { + ExpressionCompiler expressionCompiler = new ExpressionCompiler(context); + context.getResolver().setDisambiguateWithTable(true); + List ret = new ArrayList(postFilters.size()); + for (ParseNode postFilter : postFilters) { + expressionCompiler.reset(); + Expression expression = postFilter.accept(expressionCompiler); + ret.add(expression); + } + return ret; } - public List compilePostFilterExpressions() { - // TODO - return null; + public void projectColumns(Scan scan, TableRef table) { + if (isWildCardSelect(select)) { + scan.getFamilyMap().clear(); + return; + } + for (ColumnRef columnRef : columnRefs) { + if (columnRef.getTableRef().equals(table) + && !SchemaUtil.isPKColumn(columnRef.getColumn())) { + scan.addColumn(columnRef.getColumn().getFamilyName().getBytes(), columnRef.getColumn().getName().getBytes()); + } + } + } + + private class WhereNodeVisitor extends TraverseNoParseNodeVisitor { + private ColumnResolver resolver; + + public WhereNodeVisitor(ColumnResolver resolver) { + this.resolver = resolver; + } + + private Void leaveBooleanNode(ParseNode node, + List l) throws SQLException { + ColumnParseNodeVisitor visitor = new ColumnParseNodeVisitor(resolver); + node.accept(visitor); + ColumnParseNodeVisitor.ContentType type = visitor.getContentType(mainTable); + if (type == ColumnParseNodeVisitor.ContentType.NONE + || type == ColumnParseNodeVisitor.ContentType.SELF_ONLY) { + preFilters.add(node); + } else { + postFilters.add(node); + } + return null; + } + + @Override + public Void visitLeave(LikeParseNode node, + List l) throws SQLException { + return leaveBooleanNode(node, l); + } + + @Override + public boolean visitEnter(AndParseNode node) { + return true; + } + + @Override + public Void visitLeave(AndParseNode node, List l) + throws SQLException { + for (ParseNode child : node.getChildren()) { + if (child instanceof CaseParseNode) { + leaveBooleanNode(child, null); + } + } + return null; + } + + @Override + public Void visitLeave(OrParseNode node, List l) + throws SQLException { + return leaveBooleanNode(node, l); + } + + @Override + public Void visitLeave(ComparisonParseNode node, List l) + throws SQLException { + return leaveBooleanNode(node, l); + } + + @Override + public Void visitLeave(NotParseNode node, List l) + throws SQLException { + return leaveBooleanNode(node, l); + } + + @Override + public Void visitLeave(InListParseNode node, + List l) throws SQLException { + return leaveBooleanNode(node, l); + } + + @Override + public Void visitLeave(IsNullParseNode node, List l) + throws SQLException { + return leaveBooleanNode(node, l); + } } } public static JoinSpec getSubJoinSpec(JoinSpec join) { - return new JoinSpec(join.mainTable, join.select, join.preFilters, join.postFilters, join.joinTables.subList(0, join.joinTables.size() - 2), join.isPostAggregateOrDistinct, join.resolver); + return new JoinSpec(join.mainTable, join.select, join.preFilters, join.postFilters, + join.joinTables.subList(0, join.joinTables.size() - 2), join.columnRefs); } public static class JoinTable { @@ -147,16 +287,21 @@ public static class JoinTable { private List postFilters; private SelectStatement subquery; - private JoinTable(JoinType type, List conditions, TableNode tableNode, List select, - List preFilters, List postFilters, TableRef table, SelectStatement subquery) { - this.type = type; - this.conditions = conditions; - this.tableNode = tableNode; - this.select = select; - this.preFilters = preFilters; - this.postFilters = postFilters; - this.table = table; - this.subquery = subquery; + private Set leftTableRefs; + + public JoinTable(JoinTableNode node, TableRef tableRef, List select, ColumnResolver resolver) throws SQLException { + if (!(node.getTable() instanceof ConcreteTableNode)) + throw new SQLFeatureNotSupportedException(); + + this.type = node.getType(); + this.conditions = new ArrayList(); + this.tableNode = node.getTable(); + this.table = tableRef; + this.select = extractFromSelect(select,tableRef,resolver); + this.preFilters = new ArrayList(); + this.postFilters = new ArrayList(); + this.leftTableRefs = new HashSet(); + node.getOnNode().accept(new OnNodeVisitor(resolver)); } public JoinType getType() { @@ -191,6 +336,10 @@ public SelectStatement getSubquery() { return subquery; } + public Set getLeftTableRefs() { + return leftTableRefs; + } + public ParseNode getPreFiltersCombined() { if (preFilters == null || preFilters.isEmpty()) return null; @@ -211,46 +360,231 @@ public SelectStatement getAsSubquery() { } public ScanProjector getScanProjector() { - byte[] tableName = null; - if (table.getTableAlias() != null) { - tableName = Bytes.toBytes(table.getTableAlias()); - } else { - tableName = table.getTableName(); + return new ScanProjector(ProjectionType.TABLE, ScanProjector.getPrefixForTable(table), null, null); + } + + public List compilePostFilterExpressions(StatementContext context) throws SQLException { + ExpressionCompiler expressionCompiler = new ExpressionCompiler(context); + context.getResolver().setDisambiguateWithTable(true); + List ret = new ArrayList(postFilters.size()); + for (ParseNode postFilter : postFilters) { + expressionCompiler.reset(); + Expression expression = postFilter.accept(expressionCompiler); + ret.add(expression); + } + return ret; + } + + public List compileLeftTableConditions(StatementContext context) throws SQLException { + ExpressionCompiler expressionCompiler = new ExpressionCompiler(context); + context.getResolver().setDisambiguateWithTable(true); + List ret = new ArrayList(conditions.size()); + for (ParseNode condition : conditions) { + assert (condition instanceof EqualParseNode); + EqualParseNode equalNode = (EqualParseNode) condition; + expressionCompiler.reset(); + Expression expression = equalNode.getLHS().accept(expressionCompiler); + ret.add(expression); + } + return ret; + } + + public List compileRightTableConditions(StatementContext context) throws SQLException { + ExpressionCompiler expressionCompiler = new ExpressionCompiler(context); + context.getResolver().setDisambiguateWithTable(true); + List ret = new ArrayList(conditions.size()); + for (ParseNode condition : conditions) { + assert (condition instanceof EqualParseNode); + EqualParseNode equalNode = (EqualParseNode) condition; + expressionCompiler.reset(); + Expression expression = equalNode.getRHS().accept(expressionCompiler); + ret.add(expression); + } + return ret; + } + + private class OnNodeVisitor extends TraverseNoParseNodeVisitor { + private ColumnResolver resolver; + + public OnNodeVisitor(ColumnResolver resolver) { + this.resolver = resolver; + } + + private Void leaveNonEqBooleanNode(ParseNode node, + List l) throws SQLException { + ColumnParseNodeVisitor visitor = new ColumnParseNodeVisitor(resolver); + node.accept(visitor); + ColumnParseNodeVisitor.ContentType type = visitor.getContentType(table); + if (type == ColumnParseNodeVisitor.ContentType.NONE + || type == ColumnParseNodeVisitor.ContentType.SELF_ONLY) { + preFilters.add(node); + } else { + postFilters.add(node); + } + return null; + } + + @Override + public Void visitLeave(LikeParseNode node, + List l) throws SQLException { + return leaveNonEqBooleanNode(node, l); + } + + @Override + public boolean visitEnter(AndParseNode node) { + return true; + } + + @Override + public Void visitLeave(AndParseNode node, List l) + throws SQLException { + for (ParseNode child : node.getChildren()) { + if (child instanceof CaseParseNode) { + leaveNonEqBooleanNode(child, null); + } + } + return null; + } + + @Override + public Void visitLeave(OrParseNode node, List l) + throws SQLException { + return leaveNonEqBooleanNode(node, l); + } + + @Override + public Void visitLeave(ComparisonParseNode node, List l) + throws SQLException { + if (!(node instanceof EqualParseNode)) + return leaveNonEqBooleanNode(node, l); + ColumnParseNodeVisitor lhsVisitor = new ColumnParseNodeVisitor(resolver); + ColumnParseNodeVisitor rhsVisitor = new ColumnParseNodeVisitor(resolver); + node.getLHS().accept(lhsVisitor); + node.getRHS().accept(rhsVisitor); + ColumnParseNodeVisitor.ContentType lhsType = lhsVisitor.getContentType(table); + ColumnParseNodeVisitor.ContentType rhsType = rhsVisitor.getContentType(table); + if (lhsType == ColumnParseNodeVisitor.ContentType.COMPLEX) { + postFilters.add(node); + } else if (lhsType == ColumnParseNodeVisitor.ContentType.FOREIGN_ONLY) { + if (rhsType == ColumnParseNodeVisitor.ContentType.NONE + || rhsType == ColumnParseNodeVisitor.ContentType.SELF_ONLY) { + conditions.add(node); + leftTableRefs.addAll(lhsVisitor.getTableRefSet()); + } else { + postFilters.add(node); + } + } else { + if (rhsType == ColumnParseNodeVisitor.ContentType.COMPLEX) { + postFilters.add(node); + } else if (rhsType == ColumnParseNodeVisitor.ContentType.FOREIGN_ONLY) { + conditions.add(NODE_FACTORY.equal(node.getRHS(), node.getLHS())); + leftTableRefs.addAll(rhsVisitor.getTableRefSet()); + } else { + preFilters.add(node); + } + } + return null; + } + + @Override + public Void visitLeave(NotParseNode node, List l) + throws SQLException { + return leaveNonEqBooleanNode(node, l); + } + + @Override + public Void visitLeave(InListParseNode node, + List l) throws SQLException { + return leaveNonEqBooleanNode(node, l); + } + + @Override + public Void visitLeave(IsNullParseNode node, List l) + throws SQLException { + return leaveNonEqBooleanNode(node, l); } - byte[] tablePrefix = ByteUtil.concat(tableName, PROJECTION_SEPERATOR); - return new ScanProjector(ProjectionType.TABLE, tablePrefix, null, null); } + } + + private static class ColumnParseNodeVisitor extends StatelessTraverseAllParseNodeVisitor { + public enum ContentType {NONE, SELF_ONLY, FOREIGN_ONLY, COMPLEX}; - public List compilePostFilterExpressions() { - // TODO - return null; + private ColumnResolver resolver; + private final Set tableRefSet; + private final Map columnRefMap; + + public ColumnParseNodeVisitor(ColumnResolver resolver) { + this.resolver = resolver; + this.tableRefSet = new HashSet(); + this.columnRefMap = new HashMap(); } - /** - * @throws SQLException if it is not an equi-join. - */ - public List compileLeftTableConditions() throws SQLException { - // TODO - return null; + public void reset() { + this.tableRefSet.clear(); + this.columnRefMap.clear(); } - /** - * @throws SQLException if it is not an equi-join. - */ - public List compileRightTableConditions() throws SQLException { - // TODO - return null; + @Override + public Void visit(ColumnParseNode node) throws SQLException { + ColumnRef columnRef = resolver.resolveColumn(node.getSchemaName(), node.getTableName(), node.getName()); + columnRefMap.put(columnRef, node); + tableRefSet.add(columnRef.getTableRef()); + return null; + } + + public Set getTableRefSet() { + return tableRefSet; + } + + public Map getColumnRefMap() { + return columnRefMap; + } + + public ContentType getContentType(TableRef selfTable) { + if (tableRefSet.isEmpty()) + return ContentType.NONE; + if (tableRefSet.size() > 1) + return ContentType.COMPLEX; + if (tableRefSet.contains(selfTable)) + return ContentType.SELF_ONLY; + return ContentType.FOREIGN_ONLY; } } // for creation of new statements private static ParseNodeFactory NODE_FACTORY = new ParseNodeFactory(); - private static final byte[] PROJECTION_SEPERATOR = Bytes.toBytes(":"); + private static boolean isWildCardSelect(List select) { + return (select.size() == 1 && select.get(0).getNode() == WildcardParseNode.INSTANCE); + } + + private static List extractFromSelect(List select, TableRef table, ColumnResolver resolver) throws SQLException { + List ret = new ArrayList(); + if (isWildCardSelect(select)) { + ret.add(NODE_FACTORY.aliasedNode(null, WildcardParseNode.INSTANCE)); + return ret; + } + + ColumnParseNodeVisitor visitor = new ColumnParseNodeVisitor(resolver); + for (AliasedNode node : select) { + node.getNode().accept(visitor); + ColumnParseNodeVisitor.ContentType type = visitor.getContentType(table); + if (type == ColumnParseNodeVisitor.ContentType.SELF_ONLY) { + ret.add(node); + } else if (type == ColumnParseNodeVisitor.ContentType.COMPLEX) { + for (Map.Entry entry : visitor.getColumnRefMap().entrySet()) { + if (entry.getKey().getTableRef().equals(table)) { + ret.add(NODE_FACTORY.aliasedNode(null, entry.getValue())); + } + } + } + visitor.reset(); + } + return ret; + } - public static JoinSpec getJoinSpec(SelectStatement statement, PhoenixConnection connection) throws SQLException { - // TODO - return null; + public static JoinSpec getJoinSpec(StatementContext context, SelectStatement statement) throws SQLException { + return new JoinSpec(statement, context.getResolver()); } public static StarJoinType getStarJoinType(JoinSpec join) { @@ -262,15 +596,10 @@ public static StarJoinType getStarJoinType(JoinSpec join) { && joinTable.getType() != JoinType.Inner) return StarJoinType.NONE; if (starJoinType == StarJoinType.BASIC) { - List leftTableConditions; - try { - leftTableConditions = joinTable.compileLeftTableConditions(); - } catch (SQLException e) { - return StarJoinType.NONE; - } - for (Expression expr : leftTableConditions) { - // TODO test if expr consists ref to tables other than mainTable - starJoinType = StarJoinType.EXTENDED; + for (TableRef tableRef : joinTable.getLeftTableRefs()) { + if (!tableRef.equals(join.getMainTable())) { + starJoinType = StarJoinType.EXTENDED; + } } } } @@ -284,13 +613,13 @@ public static SelectStatement getSubqueryWithoutJoin(SelectStatement statement, // Get the last join table select statement with fixed-up select and where nodes. // Currently does NOT support last join table as a subquery. - public static SelectStatement getSubqueryForLastJoinTable(SelectStatement statement, JoinSpec join) { + public static SelectStatement getSubqueryForLastJoinTable(SelectStatement statement, JoinSpec join) throws SQLException { List joinTables = join.getJoinTables(); int count = joinTables.size(); assert (count > 0); JoinTable lastJoinTable = joinTables.get(count - 1); if (lastJoinTable.getSubquery() != null) { - throw new UnsupportedOperationException("Right join table cannot be a subquery."); + throw new SQLFeatureNotSupportedException("Subqueries not supported."); } List from = new ArrayList(1); from.add(lastJoinTable.getTableNode()); @@ -326,25 +655,25 @@ public static SelectStatement getSubQueryWithoutLastJoin(SelectStatement stateme // Get subquery with complete select and where nodes // Throws exception if the subquery contains joins. - public static SelectStatement getSubQueryWithoutLastJoinAsFinalPlan(SelectStatement statement, JoinSpec join) { + public static SelectStatement getSubQueryWithoutLastJoinAsFinalPlan(SelectStatement statement, JoinSpec join) throws SQLException { List from = statement.getFrom(); assert(from.size() > 1); if (from.size() > 2) - throw new UnsupportedOperationException("Left table of a left join cannot contain joins."); + throw new SQLFeatureNotSupportedException("Joins followed by a left join not supported."); return NODE_FACTORY.select(from.subList(0, from.size() - 1), statement.getHint(), statement.isDistinct(), statement.getSelect(), join.getPreFiltersCombined(), statement.getGroupBy(), statement.getHaving(), statement.getOrderBy(), statement.getLimit(), statement.getBindCount(), statement.isAggregate()); } - public static Expression getPostJoinFilterExpression(JoinSpec join, JoinTable joinTable) { + public static Expression compilePostJoinFilterExpression(StatementContext context, JoinSpec join, JoinTable joinTable) throws SQLException { List postFilters = new ArrayList(); if (joinTable != null) { - postFilters.addAll(joinTable.compilePostFilterExpressions()); + postFilters.addAll(joinTable.compilePostFilterExpressions(context)); } else { for (JoinTable table : join.getJoinTables()) { - postFilters.addAll(table.compilePostFilterExpressions()); + postFilters.addAll(table.compilePostFilterExpressions(context)); } } - postFilters.addAll(join.compilePostFilterExpressions()); + postFilters.addAll(join.compilePostFilterExpressions(context)); Expression postJoinFilterExpression = null; if (postFilters.size() == 1) { postJoinFilterExpression = postFilters.get(0); diff --git a/src/main/java/com/salesforce/phoenix/compile/PostDDLCompiler.java b/src/main/java/com/salesforce/phoenix/compile/PostDDLCompiler.java index c941e9aa..9588916b 100644 --- a/src/main/java/com/salesforce/phoenix/compile/PostDDLCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/PostDDLCompiler.java @@ -116,6 +116,11 @@ public List getTables() { public ColumnRef resolveColumn(String schemaName, String tableName, String colName) throws SQLException { throw new UnsupportedOperationException(); } + @Override + public void setDisambiguateWithTable( + boolean disambiguateWithTable) { + throw new UnsupportedOperationException(); + } }; StatementContext context = new StatementContext(SelectStatement.COUNT_ONE, connection, resolver, Collections.emptyList(), scan); ScanUtil.setTimeRange(scan, timestamp); diff --git a/src/main/java/com/salesforce/phoenix/compile/ProjectionCompiler.java b/src/main/java/com/salesforce/phoenix/compile/ProjectionCompiler.java index dc15c673..62a609b5 100644 --- a/src/main/java/com/salesforce/phoenix/compile/ProjectionCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/ProjectionCompiler.java @@ -103,7 +103,7 @@ public static RowProjector compile(StatementContext context, SelectStatement sta // Setup projected columns in Scan SelectClauseVisitor selectVisitor = new SelectClauseVisitor(context, groupBy); List projectedColumns = new ArrayList(); - TableRef tableRef = context.getResolver().getTables().get(0); + TableRef tableRef = context.getCurrentTable(); PTable table = tableRef.getTable(); boolean isWildcard = false; Scan scan = context.getScan(); @@ -118,14 +118,26 @@ public static RowProjector compile(StatementContext context, SelectStatement sta ExpressionCompiler.throwNonAggExpressionInAggException(node.toString()); } isWildcard = true; - for (int i = table.getBucketNum() == null ? 0 : 1; i < table.getColumns().size(); i++) { - ColumnRef ref = new ColumnRef(tableRef,i); - Expression expression = ref.newColumnExpression(); - projectedExpressions.add(expression); - projectedColumns.add(new ExpressionProjector(ref.getColumn().getName().getString(), table.getName().getString(), expression, false)); + boolean disambiguateWithTable = context.disambiguateWithTable(); + List tableRefs; + if (context.disambiguateWithTable()) { + tableRefs = context.getResolver().getTables(); + } else { + tableRefs = new ArrayList(); + tableRefs.add(tableRef); + } + for (TableRef tRef : tableRefs) { + PTable t = tRef.getTable(); + for (int i = t.getBucketNum() == null ? 0 : 1; i < t.getColumns().size(); i++) { + ColumnRef ref = new ColumnRef(tRef,i,disambiguateWithTable); + Expression expression = ref.newColumnExpression(); + projectedExpressions.add(expression); + projectedColumns.add(new ExpressionProjector(ref.getColumn().getName().getString(), t.getName().getString(), expression, false)); + } } } else if (node instanceof FamilyParseNode){ // Project everything for SELECT cf.* + // TODO: support cf.* expressions for multiple tables the same way with *. PColumnFamily pfamily = table.getColumnFamily(((FamilyParseNode) node).getFamilyName()); // Delay projecting to scan, as when any other column in the column family gets // added to the scan, it overwrites that we want to project the entire column @@ -164,12 +176,15 @@ public static RowProjector compile(StatementContext context, SelectStatement sta String columnAlias = aliasedNode.getAlias(); boolean isCaseSensitive = aliasedNode.isCaseSensitve() || selectVisitor.isCaseSensitive; String name = columnAlias == null ? node.toString() : columnAlias; - projectedColumns.add(new ExpressionProjector(name, table.getName().getString(), expression, isCaseSensitive)); + List tables = selectVisitor.getTables(); + String tableName = tables.isEmpty() ? table.getName().getString() : tables.get(0).getName().getString(); + projectedColumns.add(new ExpressionProjector(name, tableName, expression, isCaseSensitive)); } selectVisitor.reset(); index++; } + // TODO make estimatedByteSize more accurate by counting the joined columns. int estimatedKeySize = table.getRowKeySchema().getEstimatedValueLength(); int estimatedByteSize = 0; for (Map.Entry> entry : scan.getFamilyMap().entrySet()) { diff --git a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java index 0e964479..cbfd7805 100644 --- a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java @@ -39,7 +39,6 @@ import com.salesforce.phoenix.compile.JoinCompiler.JoinTable; import com.salesforce.phoenix.compile.JoinCompiler.StarJoinType; import com.salesforce.phoenix.compile.OrderByCompiler.OrderBy; -import com.salesforce.phoenix.coprocessor.ScanProjector; import com.salesforce.phoenix.execute.*; import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.iterate.ParallelIterators.ParallelIteratorFactory; @@ -50,7 +49,7 @@ import com.salesforce.phoenix.parse.SelectStatement; import com.salesforce.phoenix.join.HashCacheClient; import com.salesforce.phoenix.join.HashJoinInfo; -import com.salesforce.phoenix.parse.*; +import com.salesforce.phoenix.join.ScanProjector; import com.salesforce.phoenix.parse.JoinTableNode.JoinType; import com.salesforce.phoenix.query.QueryConstants; import com.salesforce.phoenix.schema.*; @@ -128,15 +127,14 @@ protected QueryPlan compile(SelectStatement statement, List binds, Scan assert(binds.size() == statement.getBindCount()); statement = StatementNormalizer.normalize(statement); - List fromNodes = statement.getFrom(); - if (fromNodes.size() == 1) { - ColumnResolver resolver = FromCompiler.getResolver(statement, connection); + ColumnResolver resolver = FromCompiler.getResolver(statement, connection); + if (statement.getFrom().size() == 1) { StatementContext context = new StatementContext(statement, connection, resolver, binds, scan); return compileSingleQuery(context, statement, binds); } - JoinSpec join = JoinCompiler.getJoinSpec(statement, connection); - StatementContext context = new StatementContext(statement, connection, join.getColumnResolver(), binds, scan, true, new HashCacheClient(connection)); + StatementContext context = new StatementContext(statement, connection, resolver, binds, scan, true, new HashCacheClient(connection)); + JoinSpec join = JoinCompiler.getJoinSpec(context, statement); return compileJoinQuery(context, statement, binds, join); } @@ -144,12 +142,15 @@ protected QueryPlan compile(SelectStatement statement, List binds, Scan protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement statement, List binds, JoinSpec join) throws SQLException { List joinTables = join.getJoinTables(); if (joinTables.isEmpty()) { + context.setCurrentTable(join.getMainTable()); + join.projectColumns(context.getScan(), join.getMainTable()); ScanProjector.serializeProjectorIntoScan(context.getScan(), join.getScanProjector()); return compileSingleQuery(context, statement, binds); } StarJoinType starJoin = JoinCompiler.getStarJoinType(join); - if (starJoin == StarJoinType.BASIC || starJoin == StarJoinType.EXTENDED) { + if (starJoin == StarJoinType.BASIC /* TODO || starJoin == StarJoinType.EXTENDED */) { + context.setCurrentTable(context.getResolver().getTables().get(0)); int count = joinTables.size(); ImmutableBytesPtr[] joinIds = new ImmutableBytesPtr[count]; List[] joinExpressions = (List[]) new List[count]; @@ -159,18 +160,19 @@ protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement s for (int i = 0; i < count; i++) { JoinTable joinTable = joinTables.get(i); joinIds[i] = new ImmutableBytesPtr(); // place-holder - joinExpressions[i] = joinTable.compileLeftTableConditions(); - hashExpressions[i] = joinTable.compileRightTableConditions(); + joinExpressions[i] = joinTable.compileLeftTableConditions(context); + hashExpressions[i] = joinTable.compileRightTableConditions(context); joinTypes[i] = joinTable.getType(); try { Scan subScan = new Scan(scanCopy); + join.projectColumns(subScan, joinTable.getTable()); ScanProjector.serializeProjectorIntoScan(subScan, joinTable.getScanProjector()); joinPlans[i] = compile(joinTable.getAsSubquery(), binds, subScan); } catch (IOException e) { throw new SQLException(e); } } - Expression postJoinFilterExpression = JoinCompiler.getPostJoinFilterExpression(join, null); + Expression postJoinFilterExpression = JoinCompiler.compilePostJoinFilterExpression(context, join, null); HashJoinInfo joinInfo = new HashJoinInfo(joinIds, joinExpressions, joinTypes, postJoinFilterExpression); HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); ScanProjector.serializeProjectorIntoScan(context.getScan(), join.getScanProjector()); @@ -181,12 +183,13 @@ protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement s JoinTable lastJoinTable = joinTables.get(joinTables.size() - 1); JoinType type = lastJoinTable.getType(); if (type == JoinType.Full) - throw new UnsupportedOperationException("Does not support full join."); + throw new SQLFeatureNotSupportedException("Full joins not supported."); if (type == JoinType.Right || (type == JoinType.Inner && joinTables.size() > 1)) { SelectStatement lhs = JoinCompiler.getSubQueryWithoutLastJoin(statement, join); SelectStatement rhs = JoinCompiler.getSubqueryForLastJoinTable(statement, join); + context.setCurrentTable(lastJoinTable.getTable()); JoinSpec lhsJoin = JoinCompiler.getSubJoinSpec(join); Scan subScan; try { @@ -194,45 +197,52 @@ protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement s } catch (IOException e) { throw new SQLException(e); } - StatementContext lhsCtx = new StatementContext(statement, connection, join.getColumnResolver(), binds, subScan, true, context.getHashClient()); + StatementContext lhsCtx = new StatementContext(statement, connection, context.getResolver(), binds, subScan, true, context.getHashClient()); QueryPlan lhsPlan = compileJoinQuery(lhsCtx, lhs, binds, lhsJoin); ImmutableBytesPtr[] joinIds = new ImmutableBytesPtr[] {new ImmutableBytesPtr()}; - Expression postJoinFilterExpression = JoinCompiler.getPostJoinFilterExpression(join, lastJoinTable); - HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {lastJoinTable.compileRightTableConditions()}, new JoinType[] {JoinType.Left}, postJoinFilterExpression); + Expression postJoinFilterExpression = JoinCompiler.compilePostJoinFilterExpression(context, join, lastJoinTable); + List joinExpressions = lastJoinTable.compileRightTableConditions(context); + List hashExpressions = lastJoinTable.compileLeftTableConditions(context); + HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {joinExpressions}, new JoinType[] {JoinType.Left}, postJoinFilterExpression); HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); ScanProjector.serializeProjectorIntoScan(context.getScan(), lastJoinTable.getScanProjector()); BasicQueryPlan rhsPlan = compileSingleQuery(context, rhs, binds); - return new HashJoinPlan(rhsPlan, joinIds, new List[] {lastJoinTable.compileLeftTableConditions()}, new QueryPlan[] {lhsPlan}); + return new HashJoinPlan(rhsPlan, joinIds, new List[] {hashExpressions}, new QueryPlan[] {lhsPlan}); } SelectStatement lhs = JoinCompiler.getSubQueryWithoutLastJoinAsFinalPlan(statement, join); SelectStatement rhs = lastJoinTable.getAsSubquery(); + context.setCurrentTable(context.getResolver().getTables().get(0)); Scan subScan; try { subScan = new Scan(scanCopy); } catch (IOException e) { throw new SQLException(e); } + join.projectColumns(subScan, lastJoinTable.getTable()); ScanProjector.serializeProjectorIntoScan(subScan, lastJoinTable.getScanProjector()); QueryPlan rhsPlan = compile(rhs, binds, subScan); ImmutableBytesPtr[] joinIds = new ImmutableBytesPtr[] {new ImmutableBytesPtr()}; - Expression postJoinFilterExpression = JoinCompiler.getPostJoinFilterExpression(join, null); - HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {lastJoinTable.compileLeftTableConditions()}, new JoinType[] {JoinType.Left}, postJoinFilterExpression); + Expression postJoinFilterExpression = JoinCompiler.compilePostJoinFilterExpression(context, join, null); + List joinExpressions = lastJoinTable.compileLeftTableConditions(context); + List hashExpressions = lastJoinTable.compileRightTableConditions(context); + HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {joinExpressions}, new JoinType[] {JoinType.Left}, postJoinFilterExpression); HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); ScanProjector.serializeProjectorIntoScan(context.getScan(), join.getScanProjector()); BasicQueryPlan lhsPlan = compileSingleQuery(context, lhs, binds); - return new HashJoinPlan(lhsPlan, joinIds, new List[] {lastJoinTable.compileRightTableConditions()}, new QueryPlan[] {rhsPlan}); + return new HashJoinPlan(lhsPlan, joinIds, new List[] {hashExpressions}, new QueryPlan[] {rhsPlan}); } protected BasicQueryPlan compileSingleQuery(StatementContext context, SelectStatement statement, List binds) throws SQLException{ ColumnResolver resolver = context.getResolver(); - TableRef tableRef = resolver.getTables().get(0); + TableRef tableRef = context.getCurrentTable(); // Short circuit out if we're compiling an index query and the index isn't active. // We must do this after the ColumnResolver resolves the table, as we may be updating the local // cache of the index table and it may now be inactive. if (tableRef.getTable().getType() == PTableType.INDEX && tableRef.getTable().getIndexState() != PIndexState.ACTIVE) { return new DegenerateQueryPlan(context, statement, tableRef); } + resolver.setDisambiguateWithTable(context.disambiguateWithTable()); Map aliasMap = ProjectionCompiler.buildAliasMap(context, statement); Integer limit = LimitCompiler.compile(context, statement); @@ -243,7 +253,9 @@ protected BasicQueryPlan compileSingleQuery(StatementContext context, SelectStat Expression having = HavingCompiler.compile(context, statement, groupBy); // Don't pass groupBy when building where clause expression, because we do not want to wrap these // expressions as group by key expressions since they're pre, not post filtered. + resolver.setDisambiguateWithTable(false); WhereCompiler.compile(context, statement); + resolver.setDisambiguateWithTable(context.disambiguateWithTable()); OrderBy orderBy = OrderByCompiler.compile(context, statement, aliasMap, groupBy, limit); RowProjector projector = ProjectionCompiler.compile(context, statement, groupBy, targetColumns); diff --git a/src/main/java/com/salesforce/phoenix/compile/StatementContext.java b/src/main/java/com/salesforce/phoenix/compile/StatementContext.java index 5333bfb0..736fdd68 100644 --- a/src/main/java/com/salesforce/phoenix/compile/StatementContext.java +++ b/src/main/java/com/salesforce/phoenix/compile/StatementContext.java @@ -69,14 +69,15 @@ public class StatementContext { private long currentTime = QueryConstants.UNSET_TIMESTAMP; private ScanRanges scanRanges = ScanRanges.EVERYTHING; - private final boolean prefixColumnFamily; + private final boolean disambiguateWithTable; private final HashCacheClient hashClient; + private TableRef currentTable; public StatementContext(BindableStatement statement, PhoenixConnection connection, ColumnResolver resolver, List binds, Scan scan) { this(statement, connection, resolver, binds, scan, false, null); } - public StatementContext(BindableStatement statement, PhoenixConnection connection, ColumnResolver resolver, List binds, Scan scan, boolean prefixColumnFamily, HashCacheClient hashClient) { + public StatementContext(BindableStatement statement, PhoenixConnection connection, ColumnResolver resolver, List binds, Scan scan, boolean disambiguateWithTable, HashCacheClient hashClient) { this.connection = connection; this.resolver = resolver; this.scan = scan; @@ -88,8 +89,9 @@ public StatementContext(BindableStatement statement, PhoenixConnection connectio this.dateParser = DateUtil.getDateParser(dateFormat); this.numberFormat = connection.getQueryServices().getProps().get(QueryServices.NUMBER_FORMAT_ATTRIB, NumberUtil.DEFAULT_NUMBER_FORMAT); this.tempPtr = new ImmutableBytesWritable(); - this.prefixColumnFamily = prefixColumnFamily; + this.disambiguateWithTable = disambiguateWithTable; this.hashClient = hashClient; + this.currentTable = resolver.getTables().get(0); } public String getDateFormat() { @@ -116,13 +118,21 @@ public BindManager getBindManager() { return binds; } - public boolean shouldPrefixColumnFamily() { - return prefixColumnFamily; + public boolean disambiguateWithTable() { + return disambiguateWithTable; } public HashCacheClient getHashClient() { return hashClient; } + + public TableRef getCurrentTable() { + return currentTable; + } + + public void setCurrentTable(TableRef table) { + this.currentTable = table; + } public AggregationManager getAggregationManager() { return aggregates; diff --git a/src/main/java/com/salesforce/phoenix/compile/WhereCompiler.java b/src/main/java/com/salesforce/phoenix/compile/WhereCompiler.java index 4cef4a22..3899affd 100644 --- a/src/main/java/com/salesforce/phoenix/compile/WhereCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/WhereCompiler.java @@ -110,12 +110,16 @@ protected ColumnRef resolveColumn(ColumnParseNode node) throws SQLException { // Track if we need to compare KeyValue during filter evaluation // using column family. If the column qualifier is enough, we // just use that. - try { - if (!SchemaUtil.isPKColumn(ref.getColumn())) { - table.getColumn(ref.getColumn().getName().getString()); - } - } catch (AmbiguousColumnException e) { + if (ref.disambiguateWithTable()) { disambiguateWithFamily = true; + } else { + try { + if (!SchemaUtil.isPKColumn(ref.getColumn())) { + table.getColumn(ref.getColumn().getName().getString()); + } + } catch (AmbiguousColumnException e) { + disambiguateWithFamily = true; + } } return ref; } @@ -140,6 +144,21 @@ public void increment(KeyValueColumnExpression column) { } } + + public void increment(RowKeyColumnExpression column) { + switch (count) { + case NONE: + count = Count.MULTIPLE; + break; + case SINGLE: + count = Count.MULTIPLE; + break; + case MULTIPLE: + break; + + } + } + public Count getCount() { return count; } @@ -175,6 +194,12 @@ public Void visit(KeyValueColumnExpression expression) { counter.increment(expression); return null; } + + @Override + public Void visit(RowKeyColumnExpression expression) { + counter.increment(expression); + return null; + } }); switch (counter.getCount()) { case NONE: diff --git a/src/main/java/com/salesforce/phoenix/coprocessor/GroupedAggregateRegionObserver.java b/src/main/java/com/salesforce/phoenix/coprocessor/GroupedAggregateRegionObserver.java index ef6e8bcf..2d127f29 100644 --- a/src/main/java/com/salesforce/phoenix/coprocessor/GroupedAggregateRegionObserver.java +++ b/src/main/java/com/salesforce/phoenix/coprocessor/GroupedAggregateRegionObserver.java @@ -50,6 +50,7 @@ import com.salesforce.phoenix.expression.aggregator.Aggregator; import com.salesforce.phoenix.expression.aggregator.ServerAggregators; import com.salesforce.phoenix.join.HashJoinInfo; +import com.salesforce.phoenix.join.ScanProjector; import com.salesforce.phoenix.memory.MemoryManager.MemoryChunk; import com.salesforce.phoenix.query.QueryConstants; import com.salesforce.phoenix.schema.tuple.MultiKeyValueTuple; diff --git a/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java b/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java index 8b05e6ca..e2d034c4 100644 --- a/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java +++ b/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java @@ -40,6 +40,7 @@ import com.salesforce.phoenix.cache.*; import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.join.HashJoinInfo; +import com.salesforce.phoenix.join.ScanProjector; import com.salesforce.phoenix.parse.JoinTableNode.JoinType; import com.salesforce.phoenix.schema.IllegalDataException; import com.salesforce.phoenix.schema.tuple.ResultTuple; diff --git a/src/main/java/com/salesforce/phoenix/coprocessor/ScanRegionObserver.java b/src/main/java/com/salesforce/phoenix/coprocessor/ScanRegionObserver.java index 9d8276d8..93de5947 100644 --- a/src/main/java/com/salesforce/phoenix/coprocessor/ScanRegionObserver.java +++ b/src/main/java/com/salesforce/phoenix/coprocessor/ScanRegionObserver.java @@ -47,6 +47,7 @@ import com.salesforce.phoenix.expression.OrderByExpression; import com.salesforce.phoenix.iterate.*; import com.salesforce.phoenix.join.HashJoinInfo; +import com.salesforce.phoenix.join.ScanProjector; import com.salesforce.phoenix.memory.MemoryManager.MemoryChunk; import com.salesforce.phoenix.schema.PDataType; import com.salesforce.phoenix.schema.tuple.Tuple; @@ -132,7 +133,6 @@ protected RegionScanner doPostScannerOpen(final ObserverContext cfProjectionMap; private final Map>> cqProjectionMap; + + public static byte[] getPrefixedColumnFamily(byte[] columnFamily, byte[] cfPrefix) { + return ByteUtil.concat(cfPrefix, PREFIX_SEPERATOR, columnFamily); + } + + public static int getPrefixLength(byte[] columnFamily) { + return getPrefixLength(columnFamily, 0, columnFamily.length); + } + + public static int getPrefixLength(byte[] cfBuffer, int offset, int length) { + for (int i = offset + length - 1; i >= offset; i--) { + if (cfBuffer[i] == PREFIX_SEPERATOR[0]) { + return (i - offset); + } + } + return 0; + } + + public static byte[] getPrefixForTable(TableRef table) { + if (table.getTableAlias() == null) + return table.getTableName(); + + return Bytes.toBytes(table.getTableAlias()); + } public ScanProjector(ProjectionType type, byte[] tablePrefix, Map cfProjectionMap, Map>> getC public KeyValue getProjectedKeyValue(KeyValue kv) { if (type == ProjectionType.TABLE) { - byte[] cf = ByteUtil.concat(tablePrefix, kv.getFamily()); + byte[] cf = getPrefixedColumnFamily(kv.getFamily(), tablePrefix); return KeyValueUtil.newKeyValue(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength(), cf, kv.getQualifier(), kv.getTimestamp(), kv.getBuffer(), kv.getValueOffset(), kv.getValueLength()); } diff --git a/src/main/java/com/salesforce/phoenix/schema/ColumnRef.java b/src/main/java/com/salesforce/phoenix/schema/ColumnRef.java index cc98c747..7f16e390 100644 --- a/src/main/java/com/salesforce/phoenix/schema/ColumnRef.java +++ b/src/main/java/com/salesforce/phoenix/schema/ColumnRef.java @@ -32,6 +32,7 @@ import org.apache.http.annotation.Immutable; import com.salesforce.phoenix.expression.*; +import com.salesforce.phoenix.join.ScanProjector; import com.salesforce.phoenix.util.SchemaUtil; @@ -47,8 +48,13 @@ public final class ColumnRef { private final TableRef tableRef; private final int columnPosition; private final int pkSlotPosition; + private final boolean disambiguateWithTable; public ColumnRef(TableRef tableRef, int columnPosition) { + this(tableRef, columnPosition, false); + } + + public ColumnRef(TableRef tableRef, int columnPosition, boolean disambiguateWithTable) { if (tableRef == null) { throw new NullPointerException(); } @@ -68,6 +74,7 @@ public ColumnRef(TableRef tableRef, int columnPosition) { } } pkSlotPosition = i; + this.disambiguateWithTable = disambiguateWithTable; } @Override @@ -92,10 +99,18 @@ public boolean equals(Object obj) { public ColumnExpression newColumnExpression() throws SQLException { if (SchemaUtil.isPKColumn(this.getColumn())) { + if (disambiguateWithTable) { + return new RowKeyColumnExpression(getColumn(), new RowKeyValueAccessor(this.getTable().getPKColumns(), pkSlotPosition), ScanProjector.getPrefixForTable(tableRef)); + } + return new RowKeyColumnExpression(getColumn(), new RowKeyValueAccessor(this.getTable().getPKColumns(), pkSlotPosition)); - } else { - return new KeyValueColumnExpression(getColumn()); } + + if (disambiguateWithTable) { + return new KeyValueColumnExpression(getColumn(), ScanProjector.getPrefixForTable(tableRef)); + } + + return new KeyValueColumnExpression(getColumn()); } public int getColumnPosition() { @@ -121,4 +136,8 @@ public PSchema getSchema() { public TableRef getTableRef() { return tableRef; } + + public boolean disambiguateWithTable() { + return disambiguateWithTable; + } } diff --git a/src/main/java/com/salesforce/phoenix/schema/tuple/MultiKeyValueTuple.java b/src/main/java/com/salesforce/phoenix/schema/tuple/MultiKeyValueTuple.java index 650f7e7a..59a0345c 100644 --- a/src/main/java/com/salesforce/phoenix/schema/tuple/MultiKeyValueTuple.java +++ b/src/main/java/com/salesforce/phoenix/schema/tuple/MultiKeyValueTuple.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; import com.salesforce.phoenix.util.KeyValueUtil; @@ -79,4 +80,17 @@ public int size() { public KeyValue getValue(int index) { return values.get(index); } + + @Override + public boolean getKey(ImmutableBytesWritable ptr, byte[] cfPrefix) { + for (KeyValue kv : values) { + int len = kv.getFamilyLength(); + if (len >= cfPrefix.length + && Bytes.equals(cfPrefix, 0, cfPrefix.length, kv.getBuffer(), kv.getFamilyOffset(), len)) { + ptr.set(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength()); + return true; + } + } + return false; + } } diff --git a/src/main/java/com/salesforce/phoenix/schema/tuple/ResultTuple.java b/src/main/java/com/salesforce/phoenix/schema/tuple/ResultTuple.java index 3dbe9abf..4ec39cf7 100644 --- a/src/main/java/com/salesforce/phoenix/schema/tuple/ResultTuple.java +++ b/src/main/java/com/salesforce/phoenix/schema/tuple/ResultTuple.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; import com.salesforce.phoenix.util.ResultUtil; @@ -81,4 +82,17 @@ public int size() { public KeyValue getValue(int index) { return result.raw()[index]; } + + @Override + public boolean getKey(ImmutableBytesWritable ptr, byte[] cfPrefix) { + for (KeyValue kv : result.raw()) { + int len = kv.getFamilyLength(); + if (len >= cfPrefix.length + && Bytes.equals(cfPrefix, 0, cfPrefix.length, kv.getBuffer(), kv.getFamilyOffset(), len)) { + ptr.set(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength()); + return true; + } + } + return false; + } } diff --git a/src/main/java/com/salesforce/phoenix/schema/tuple/SingleKeyValueTuple.java b/src/main/java/com/salesforce/phoenix/schema/tuple/SingleKeyValueTuple.java index 6dc49a70..ccf7a94a 100644 --- a/src/main/java/com/salesforce/phoenix/schema/tuple/SingleKeyValueTuple.java +++ b/src/main/java/com/salesforce/phoenix/schema/tuple/SingleKeyValueTuple.java @@ -107,4 +107,15 @@ public KeyValue getValue(int index) { } return keyValue; } + + @Override + public boolean getKey(ImmutableBytesWritable ptr, byte[] cfPrefix) { + int len = keyValue.getFamilyLength(); + if (len >= cfPrefix.length + && Bytes.equals(cfPrefix, 0, cfPrefix.length, keyValue.getBuffer(), keyValue.getFamilyOffset(), len)) { + ptr.set(keyValue.getBuffer(), keyValue.getKeyOffset(), keyValue.getKeyLength()); + return true; + } + return false; + } } diff --git a/src/main/java/com/salesforce/phoenix/schema/tuple/Tuple.java b/src/main/java/com/salesforce/phoenix/schema/tuple/Tuple.java index 6858d5b4..352486c6 100644 --- a/src/main/java/com/salesforce/phoenix/schema/tuple/Tuple.java +++ b/src/main/java/com/salesforce/phoenix/schema/tuple/Tuple.java @@ -60,6 +60,18 @@ public interface Tuple { */ public void getKey(ImmutableBytesWritable ptr); + /** + * Get the row key for the Tuple where the row key is contained + * in a KeyValue having the specified column family prefix. + * This is used for searching a joined result for a row key from + * a specific table. + * @param ptr the bytes pointer that will be updated to point to + * the key buffer. + * @param cfPrefix the column family prefix. + * @return true if the specified KeyValue is found. + */ + public boolean getKey(ImmutableBytesWritable ptr, byte[] cfPrefix); + /** * Get the KeyValue at the given index. * @param index the zero-based KeyValue index between 0 and {@link #size()} exclusive From 4313291ec2c38bc47abc2532a6e7b2f151d5bd48 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Mon, 16 Sep 2013 01:14:55 -0700 Subject: [PATCH 027/102] Minor fixes to MD5 function --- .../phoenix/execute/MutationState.java | 3 ++ .../expression/function/MD5Function.java | 30 ++++++++++++++----- .../phoenix/end2end/MD5FunctionTest.java | 30 ++++++++++++++----- 3 files changed, 47 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/execute/MutationState.java b/src/main/java/com/salesforce/phoenix/execute/MutationState.java index 697afd04..ae279786 100644 --- a/src/main/java/com/salesforce/phoenix/execute/MutationState.java +++ b/src/main/java/com/salesforce/phoenix/execute/MutationState.java @@ -359,7 +359,10 @@ public void commit() throws SQLException { HTableInterface hTable = connection.getQueryServices().getTable(htableName); try { if (logger.isDebugEnabled()) logMutationSize(hTable, mutations); + long startTime = System.currentTimeMillis(); hTable.batch(mutations); + long totalTime = System.currentTimeMillis() - startTime; + System.err.println("Total time for commit of " + mutations.size() + " rows: " + totalTime + " ms"); committedList.add(entry); } catch (Exception e) { // Throw to client with both what was committed so far and what is left to be committed. diff --git a/src/main/java/com/salesforce/phoenix/expression/function/MD5Function.java b/src/main/java/com/salesforce/phoenix/expression/function/MD5Function.java index 8e8a7152..9688ed85 100644 --- a/src/main/java/com/salesforce/phoenix/expression/function/MD5Function.java +++ b/src/main/java/com/salesforce/phoenix/expression/function/MD5Function.java @@ -35,16 +35,26 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import com.salesforce.phoenix.expression.Expression; -import com.salesforce.phoenix.parse.FunctionParseNode.*; +import com.salesforce.phoenix.parse.FunctionParseNode.Argument; +import com.salesforce.phoenix.parse.FunctionParseNode.BuiltInFunction; import com.salesforce.phoenix.schema.PDataType; import com.salesforce.phoenix.schema.tuple.Tuple; @BuiltInFunction(name = MD5Function.NAME, args={@Argument()}) public class MD5Function extends ScalarFunction { public static final String NAME = "MD5"; + public static final Integer LENGTH = 16; private final MessageDigest messageDigest; + public MD5Function() throws SQLException { + try { + messageDigest = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + throw new SQLException(e); + } + } + public MD5Function(List children) throws SQLException { super(children); try { @@ -53,19 +63,13 @@ public MD5Function(List children) throws SQLException { throw new RuntimeException(e); } } - + @Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { if (!getChildExpression().evaluate(tuple, ptr)) { return false; } - String sourceStr = (String) PDataType.VARCHAR.toObject(ptr, getChildExpression().getColumnModifier()); - - if (sourceStr == null) { - return true; - } - // Update the digest value messageDigest.update(ptr.get(), ptr.getOffset(), ptr.getLength()); // Get the digest bytes (note this resets the messageDigest as well) @@ -78,6 +82,16 @@ public PDataType getDataType() { return PDataType.BINARY; } + @Override + public Integer getMaxLength() { + return LENGTH; + } + + @Override + public Integer getByteSize() { + return LENGTH; + } + @Override public boolean isNullable() { return getChildExpression().isNullable(); diff --git a/src/test/java/com/salesforce/phoenix/end2end/MD5FunctionTest.java b/src/test/java/com/salesforce/phoenix/end2end/MD5FunctionTest.java index ae6b23d4..3a5190d5 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/MD5FunctionTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/MD5FunctionTest.java @@ -27,11 +27,14 @@ ******************************************************************************/ package com.salesforce.phoenix.end2end; -import static org.junit.Assert.*; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import java.security.MessageDigest; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.PreparedStatement; import java.sql.ResultSet; import org.junit.Test; @@ -59,20 +62,31 @@ public void testRetrieve() throws Exception { @Test public void testUpsert() throws Exception { - String testString = "mwalsh"; + String testString1 = "mwalsh1"; + String testString2 = "mwalsh2"; Connection conn = DriverManager.getConnection(getUrl()); - String ddl = "CREATE TABLE IF NOT EXISTS MD5_UPSERT_TEST (pk binary(16) NOT NULL PRIMARY KEY)"; + String ddl = "CREATE TABLE IF NOT EXISTS MD5_UPSERT_TEST (k1 binary(16) NOT NULL,k2 binary(16) NOT NULL CONSTRAINT pk PRIMARY KEY (k1, k2))"; conn.createStatement().execute(ddl); - String dml = String.format("UPSERT INTO MD5_UPSERT_TEST VALUES(md5('%s'))", testString); + String dml = String.format("UPSERT INTO MD5_UPSERT_TEST VALUES(md5('%s'),md5('%s'))", testString1, testString2); conn.createStatement().execute(dml); conn.commit(); - ResultSet rs = conn.createStatement().executeQuery("SELECT pk FROM MD5_UPSERT_TEST"); + ResultSet rs = conn.createStatement().executeQuery("SELECT k1,k2 FROM MD5_UPSERT_TEST"); assertTrue(rs.next()); - byte[] first = MessageDigest.getInstance("MD5").digest(testString.getBytes()); - byte[] second = rs.getBytes(1); - assertArrayEquals(first, second); + byte[] pk1 = MessageDigest.getInstance("MD5").digest(testString1.getBytes()); + byte[] pk2 = MessageDigest.getInstance("MD5").digest(testString2.getBytes()); + assertArrayEquals(pk1, rs.getBytes(1)); + assertArrayEquals(pk2, rs.getBytes(2)); + assertFalse(rs.next()); + PreparedStatement stmt = conn.prepareStatement("SELECT k1,k2 FROM MD5_UPSERT_TEST WHERE k1=md5(?)"); + stmt.setString(1, testString1); + rs = stmt.executeQuery(); + assertTrue(rs.next()); + byte[] second1 = rs.getBytes(1); + byte[] second2 = rs.getBytes(2); + assertArrayEquals(pk1, second1); + assertArrayEquals(pk2, second2); assertFalse(rs.next()); } From 69b6ee513862efb3483b9916dee3976e7522be12 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Mon, 16 Sep 2013 12:00:18 -0700 Subject: [PATCH 028/102] Fix upgrade code to not be triggered when it shouldnt, Add logging for time to process htable.batch,Fix compiler warnings --- pom.xml | 2 +- .../example/CoveredColumnIndexCodec.java | 1 - .../phoenix/compile/FromCompiler.java | 7 +-- .../phoenix/coprocessor/MetaDataProtocol.java | 6 ++- .../phoenix/exception/SQLExceptionInfo.java | 2 +- .../phoenix/execute/MutationState.java | 3 +- .../phoenix/expression/ColumnExpression.java | 12 +++-- .../phoenix/expression/ExpressionType.java | 34 +++++++++++-- .../phoenix/jdbc/PhoenixDatabaseMetaData.java | 3 +- .../query/ConnectionQueryServicesImpl.java | 24 +++++----- .../schema/SchemaNotFoundException.java | 48 ------------------- .../salesforce/phoenix/util/SchemaUtil.java | 26 +++++----- .../phoenix/schema/SchemaUtilTest.java | 10 ++-- .../com/salesforce/phoenix/util/TestUtil.java | 2 +- 14 files changed, 82 insertions(+), 98 deletions(-) delete mode 100644 src/main/java/com/salesforce/phoenix/schema/SchemaNotFoundException.java diff --git a/pom.xml b/pom.xml index 8e9d24ea..f6d2218d 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.salesforce phoenix - 2.1.0-SNAPSHOT + 2.2.0-SNAPSHOT Phoenix A SQL layer over HBase diff --git a/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexCodec.java b/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexCodec.java index c9e51919..76991f77 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexCodec.java +++ b/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexCodec.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Iterator; import java.util.List; import java.util.Map.Entry; diff --git a/src/main/java/com/salesforce/phoenix/compile/FromCompiler.java b/src/main/java/com/salesforce/phoenix/compile/FromCompiler.java index b2ecba13..ff32c4f8 100644 --- a/src/main/java/com/salesforce/phoenix/compile/FromCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/FromCompiler.java @@ -55,7 +55,6 @@ import com.salesforce.phoenix.schema.PNameFactory; import com.salesforce.phoenix.schema.PTable; import com.salesforce.phoenix.schema.PTableImpl; -import com.salesforce.phoenix.schema.SchemaNotFoundException; import com.salesforce.phoenix.schema.TableNotFoundException; import com.salesforce.phoenix.schema.TableRef; import com.salesforce.phoenix.util.SchemaUtil; @@ -150,10 +149,8 @@ public SingleTableColumnResolver(PhoenixConnection connection, NamedTableNode ta logger.debug("Re-resolved stale table " + fullTableName + " with seqNum " + tableRef.getTable().getSequenceNumber() + " at timestamp " + tableRef.getTable().getTimeStamp() + " with " + tableRef.getTable().getColumns().size() + " columns: " + tableRef.getTable().getColumns()); } break; - } catch (SchemaNotFoundException e) { - sqlE = new TableNotFoundException(schemaName, tableName); } catch (TableNotFoundException e) { - sqlE = new TableNotFoundException(schemaName, tableName); + sqlE = e; } if (retry && client.updateCache(schemaName, tableName) < 0) { retry = false; @@ -300,7 +297,7 @@ private TableRef resolveTable(String schemaName, String tableName) throws SQLExc String fullTableName = SchemaUtil.getTableName(schemaName, tableName); List tableRefs = tableMap.get(fullTableName); if (tableRefs.size() == 0) { - throw new TableNotFoundException(schemaName, tableName); + throw new TableNotFoundException(fullTableName); } else if (tableRefs.size() > 1) { throw new AmbiguousTableException(tableName); } else { diff --git a/src/main/java/com/salesforce/phoenix/coprocessor/MetaDataProtocol.java b/src/main/java/com/salesforce/phoenix/coprocessor/MetaDataProtocol.java index b9b42558..7031bd1d 100644 --- a/src/main/java/com/salesforce/phoenix/coprocessor/MetaDataProtocol.java +++ b/src/main/java/com/salesforce/phoenix/coprocessor/MetaDataProtocol.java @@ -27,7 +27,9 @@ ******************************************************************************/ package com.salesforce.phoenix.coprocessor; -import java.io.*; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; import java.util.List; import org.apache.hadoop.hbase.client.Mutation; @@ -59,7 +61,7 @@ */ public interface MetaDataProtocol extends CoprocessorProtocol { public static final int PHOENIX_MAJOR_VERSION = 2; - public static final int PHOENIX_MINOR_VERSION = 1; + public static final int PHOENIX_MINOR_VERSION = 2; public static final int PHOENIX_PATCH_NUMBER = 0; public static final int PHOENIX_VERSION = MetaDataUtil.encodeVersion(PHOENIX_MAJOR_VERSION, PHOENIX_MINOR_VERSION, PHOENIX_PATCH_NUMBER); diff --git a/src/main/java/com/salesforce/phoenix/exception/SQLExceptionInfo.java b/src/main/java/com/salesforce/phoenix/exception/SQLExceptionInfo.java index 0249eba4..513dbd0c 100644 --- a/src/main/java/com/salesforce/phoenix/exception/SQLExceptionInfo.java +++ b/src/main/java/com/salesforce/phoenix/exception/SQLExceptionInfo.java @@ -126,7 +126,7 @@ public String toString() { if (message != null) { builder.append(" ").append(message); } - String columnDisplayName = SchemaUtil.getColumnName(schemaName, tableName, familyName, columnName); + String columnDisplayName = SchemaUtil.getMetaDataEntityName(schemaName, tableName, familyName, columnName); if (columnName != null) { builder.append(" ").append(COLUMN_NAME).append("=").append(columnDisplayName); } else if (familyName != null) { diff --git a/src/main/java/com/salesforce/phoenix/execute/MutationState.java b/src/main/java/com/salesforce/phoenix/execute/MutationState.java index ae279786..94143e58 100644 --- a/src/main/java/com/salesforce/phoenix/execute/MutationState.java +++ b/src/main/java/com/salesforce/phoenix/execute/MutationState.java @@ -361,8 +361,7 @@ public void commit() throws SQLException { if (logger.isDebugEnabled()) logMutationSize(hTable, mutations); long startTime = System.currentTimeMillis(); hTable.batch(mutations); - long totalTime = System.currentTimeMillis() - startTime; - System.err.println("Total time for commit of " + mutations.size() + " rows: " + totalTime + " ms"); + if (logger.isDebugEnabled()) logger.debug("Total time for batch call of " + mutations.size() + " mutations into " + table.getName().getString() + ": " + (System.currentTimeMillis() - startTime) + " ms"); committedList.add(entry); } catch (Exception e) { // Throw to client with both what was committed so far and what is left to be committed. diff --git a/src/main/java/com/salesforce/phoenix/expression/ColumnExpression.java b/src/main/java/com/salesforce/phoenix/expression/ColumnExpression.java index 4987add5..ba78d086 100644 --- a/src/main/java/com/salesforce/phoenix/expression/ColumnExpression.java +++ b/src/main/java/com/salesforce/phoenix/expression/ColumnExpression.java @@ -27,12 +27,16 @@ ******************************************************************************/ package com.salesforce.phoenix.expression; -import java.io.*; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; import org.apache.hadoop.io.WritableUtils; import com.google.common.base.Objects; -import com.salesforce.phoenix.schema.*; +import com.salesforce.phoenix.schema.ColumnModifier; +import com.salesforce.phoenix.schema.PDataType; +import com.salesforce.phoenix.schema.PDatum; /** * @@ -141,7 +145,7 @@ public void readFields(DataInput input) throws IOException { @Override public void write(DataOutput output) throws IOException { // read/write type ordinal, maxLength presence, scale presence and isNullable bit together to save space - int typeAndFlag = (isNullable ? 1 : 0) | (scale != null ? 1 : 0) << 1 | (maxLength != null ? 1 : 0) << 2 + int typeAndFlag = (isNullable ? 1 : 0) | ((scale != null ? 1 : 0) << 1) | ((maxLength != null ? 1 : 0) << 2) | (type.ordinal() << 3); WritableUtils.writeVInt(output,typeAndFlag); if (scale != null) { @@ -150,7 +154,7 @@ public void write(DataOutput output) throws IOException { if (maxLength != null) { WritableUtils.writeVInt(output, maxLength); } - if (byteSize != null) { + if (type.isFixedWidth() && type.getByteSize() == null) { WritableUtils.writeVInt(output, byteSize); } WritableUtils.writeVInt(output, ColumnModifier.toSystemValue(columnModifier)); diff --git a/src/main/java/com/salesforce/phoenix/expression/ExpressionType.java b/src/main/java/com/salesforce/phoenix/expression/ExpressionType.java index 2181cfd0..ce70f94c 100644 --- a/src/main/java/com/salesforce/phoenix/expression/ExpressionType.java +++ b/src/main/java/com/salesforce/phoenix/expression/ExpressionType.java @@ -30,7 +30,34 @@ import java.util.Map; import com.google.common.collect.Maps; -import com.salesforce.phoenix.expression.function.*; +import com.salesforce.phoenix.expression.function.CoalesceFunction; +import com.salesforce.phoenix.expression.function.CountAggregateFunction; +import com.salesforce.phoenix.expression.function.DistinctCountAggregateFunction; +import com.salesforce.phoenix.expression.function.LTrimFunction; +import com.salesforce.phoenix.expression.function.LengthFunction; +import com.salesforce.phoenix.expression.function.LowerFunction; +import com.salesforce.phoenix.expression.function.MD5Function; +import com.salesforce.phoenix.expression.function.MaxAggregateFunction; +import com.salesforce.phoenix.expression.function.MinAggregateFunction; +import com.salesforce.phoenix.expression.function.PercentRankAggregateFunction; +import com.salesforce.phoenix.expression.function.PercentileContAggregateFunction; +import com.salesforce.phoenix.expression.function.PercentileDiscAggregateFunction; +import com.salesforce.phoenix.expression.function.RTrimFunction; +import com.salesforce.phoenix.expression.function.RegexpReplaceFunction; +import com.salesforce.phoenix.expression.function.RegexpSubstrFunction; +import com.salesforce.phoenix.expression.function.ReverseFunction; +import com.salesforce.phoenix.expression.function.RoundFunction; +import com.salesforce.phoenix.expression.function.SqlTypeNameFunction; +import com.salesforce.phoenix.expression.function.StddevPopFunction; +import com.salesforce.phoenix.expression.function.StddevSampFunction; +import com.salesforce.phoenix.expression.function.SubstrFunction; +import com.salesforce.phoenix.expression.function.SumAggregateFunction; +import com.salesforce.phoenix.expression.function.ToCharFunction; +import com.salesforce.phoenix.expression.function.ToDateFunction; +import com.salesforce.phoenix.expression.function.ToNumberFunction; +import com.salesforce.phoenix.expression.function.TrimFunction; +import com.salesforce.phoenix.expression.function.TruncFunction; +import com.salesforce.phoenix.expression.function.UpperFunction; /** * @@ -42,7 +69,6 @@ * @since 0.1 */ public enum ExpressionType { - MD5Function(MD5Function.class), ReverseFunction(ReverseFunction.class), RowKey(RowKeyColumnExpression.class), KeyValue(KeyValueColumnExpression.class), @@ -96,7 +122,9 @@ public enum ExpressionType { DoubleAddExpression(DoubleAddExpression.class), DoubleSubtractExpression(DoubleSubtractExpression.class), DoubleMultiplyExpression(DoubleMultiplyExpression.class), - DoubleDivideExpression(DoubleDivideExpression.class); + DoubleDivideExpression(DoubleDivideExpression.class), + MD5Function(MD5Function.class); + ExpressionType(Class clazz) { this.clazz = clazz; } diff --git a/src/main/java/com/salesforce/phoenix/jdbc/PhoenixDatabaseMetaData.java b/src/main/java/com/salesforce/phoenix/jdbc/PhoenixDatabaseMetaData.java index adf4b76f..c436ac4a 100644 --- a/src/main/java/com/salesforce/phoenix/jdbc/PhoenixDatabaseMetaData.java +++ b/src/main/java/com/salesforce/phoenix/jdbc/PhoenixDatabaseMetaData.java @@ -96,7 +96,8 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData, com.salesforce public static final String TYPE_SCHEMA_AND_TABLE = TYPE_SCHEMA + ".\"" + TYPE_TABLE + "\""; public static final byte[] TYPE_TABLE_BYTES = TYPE_TABLE.getBytes(); public static final byte[] TYPE_SCHEMA_BYTES = TYPE_SCHEMA.getBytes(); - public static final byte[] TYPE_TABLE_NAME = SchemaUtil.getTableNameAsBytes(TYPE_SCHEMA_BYTES, TYPE_TABLE_BYTES); + public static final String TYPE_TABLE_NAME = SchemaUtil.getTableName(TYPE_SCHEMA, TYPE_TABLE); + public static final byte[] TYPE_TABLE_NAME_BYTES = SchemaUtil.getTableNameAsBytes(TYPE_SCHEMA_BYTES, TYPE_TABLE_BYTES); public static final String TABLE_NAME_NAME = "TABLE_NAME"; public static final String TABLE_TYPE_NAME = "TABLE_TYPE"; diff --git a/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java b/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java index ce6b35e2..0b32fec3 100644 --- a/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java +++ b/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java @@ -31,7 +31,7 @@ import static com.salesforce.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TABLE_NAME_BYTES; import static com.salesforce.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES; import static com.salesforce.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE_BYTES; -import static com.salesforce.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_TABLE_NAME; +import static com.salesforce.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_TABLE_NAME_BYTES; import static com.salesforce.phoenix.util.SchemaUtil.getVarChars; import java.io.IOException; @@ -119,7 +119,6 @@ import com.salesforce.phoenix.schema.PTableType; import com.salesforce.phoenix.schema.ReadOnlyTableException; import com.salesforce.phoenix.schema.SaltingUtil; -import com.salesforce.phoenix.schema.SchemaNotFoundException; import com.salesforce.phoenix.schema.TableAlreadyExistsException; import com.salesforce.phoenix.schema.TableNotFoundException; import com.salesforce.phoenix.schema.TableRef; @@ -345,7 +344,6 @@ private PMetaData metaDataMutated(String tableName, long tableSeqNum, Mutator mu logger.warn("Attempt to cache older version of " + tableName + ": current= " + table.getSequenceNumber() + ", new=" + tableSeqNum); break; } - } catch (SchemaNotFoundException e) { } catch (TableNotFoundException e) { } long waitTime = endTime - System.currentTimeMillis(); @@ -745,7 +743,7 @@ private void upgradeTablesFrom0_94_2to0_94_4(HBaseAdmin admin) throws IOExceptio scan.addColumn(TABLE_FAMILY_BYTES, COLUMN_COUNT_BYTES); // Add filter so that we only get the table row and not the column rows scan.setFilter(new SingleColumnValueFilter(TABLE_FAMILY_BYTES, COLUMN_COUNT_BYTES, CompareOp.GREATER_OR_EQUAL, PDataType.INTEGER.toBytes(0))); - HTableInterface table = HBaseFactoryProvider.getHTableFactory().getTable(TYPE_TABLE_NAME, connection, getExecutor()); + HTableInterface table = HBaseFactoryProvider.getHTableFactory().getTable(TYPE_TABLE_NAME_BYTES, connection, getExecutor()); ResultScanner scanner = table.getScanner(scan); Result result = null; while ((result = scanner.next()) != null) { @@ -805,7 +803,7 @@ private void upgradeTablesFrom2_0to2_1(HBaseAdmin admin, HTableDescriptor newDes filter.setFilterIfMissing(true); // Add filter so that we only get the table row and not the column rows scan.setFilter(filter); - HTableInterface table = HBaseFactoryProvider.getHTableFactory().getTable(TYPE_TABLE_NAME, connection, getExecutor()); + HTableInterface table = HBaseFactoryProvider.getHTableFactory().getTable(TYPE_TABLE_NAME_BYTES, connection, getExecutor()); ResultScanner scanner = table.getScanner(scan); Result result = null; List indexesToUpdate = Lists.newArrayList(); @@ -920,9 +918,9 @@ private void upgradeTablesFrom2_0to2_1(HBaseAdmin admin, HTableDescriptor newDes // Finally, at the end, modify the system table // which will include the flag that'll prevent this // update from occurring again - admin.disableTable(TYPE_TABLE_NAME); - admin.modifyTable(TYPE_TABLE_NAME, newDesc); - admin.enableTable(TYPE_TABLE_NAME); + admin.disableTable(TYPE_TABLE_NAME_BYTES); + admin.modifyTable(TYPE_TABLE_NAME_BYTES, newDesc); + admin.enableTable(TYPE_TABLE_NAME_BYTES); if (logger.isInfoEnabled()) { logger.info("Upgrade to 2.1.0 completed successfully" ); } @@ -940,7 +938,7 @@ private void checkClientServerCompatibility() throws SQLException { boolean isIncompatible = false; int minHBaseVersion = Integer.MAX_VALUE; try { - NavigableMap regionInfoMap = MetaScanner.allTableRegions(config, TYPE_TABLE_NAME, false); + NavigableMap regionInfoMap = MetaScanner.allTableRegions(config, TYPE_TABLE_NAME_BYTES, false); Set serverMap = Sets.newHashSetWithExpectedSize(regionInfoMap.size()); TreeMap regionMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); List regionKeys = Lists.newArrayListWithExpectedSize(regionInfoMap.size()); @@ -953,7 +951,7 @@ private void checkClientServerCompatibility() throws SQLException { } final TreeMap results = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); connection.processExecs(MetaDataProtocol.class, regionKeys, - PhoenixDatabaseMetaData.TYPE_TABLE_NAME, this.getDelegate().getExecutor(), new Batch.Call() { + PhoenixDatabaseMetaData.TYPE_TABLE_NAME_BYTES, this.getDelegate().getExecutor(), new Batch.Call() { @Override public Long call(MetaDataProtocol instance) throws IOException { return instance.getVersion(); @@ -998,11 +996,11 @@ private MetaDataMutationResult metaDataCoprocessorExec(byte[] tableKey, Batch.Ca try { boolean retried = false; while (true) { - HRegionLocation regionLocation = retried ? connection.relocateRegion(PhoenixDatabaseMetaData.TYPE_TABLE_NAME, tableKey) : connection.locateRegion(PhoenixDatabaseMetaData.TYPE_TABLE_NAME, tableKey); + HRegionLocation regionLocation = retried ? connection.relocateRegion(PhoenixDatabaseMetaData.TYPE_TABLE_NAME_BYTES, tableKey) : connection.locateRegion(PhoenixDatabaseMetaData.TYPE_TABLE_NAME_BYTES, tableKey); List regionKeys = Collections.singletonList(regionLocation.getRegionInfo().getStartKey()); final Map results = Maps.newHashMapWithExpectedSize(1); connection.processExecs(MetaDataProtocol.class, regionKeys, - PhoenixDatabaseMetaData.TYPE_TABLE_NAME, this.getDelegate().getExecutor(), callable, + PhoenixDatabaseMetaData.TYPE_TABLE_NAME_BYTES, this.getDelegate().getExecutor(), callable, new Batch.Callback(){ @Override public void update(byte[] region, byte[] row, MetaDataMutationResult value) { @@ -1163,7 +1161,7 @@ public int getLowestClusterHBaseVersion() { protected void clearCache() throws SQLException { try { SQLException sqlE = null; - HTableInterface htable = this.getTable(PhoenixDatabaseMetaData.TYPE_TABLE_NAME); + HTableInterface htable = this.getTable(PhoenixDatabaseMetaData.TYPE_TABLE_NAME_BYTES); try { htable.coprocessorExec(MetaDataProtocol.class, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, new Batch.Call() { diff --git a/src/main/java/com/salesforce/phoenix/schema/SchemaNotFoundException.java b/src/main/java/com/salesforce/phoenix/schema/SchemaNotFoundException.java deleted file mode 100644 index f8d7d50a..00000000 --- a/src/main/java/com/salesforce/phoenix/schema/SchemaNotFoundException.java +++ /dev/null @@ -1,48 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2013, Salesforce.com, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of Salesforce.com nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ******************************************************************************/ -package com.salesforce.phoenix.schema; - -import com.salesforce.phoenix.exception.SQLExceptionCode; -import com.salesforce.phoenix.exception.SQLExceptionInfo; - -public class SchemaNotFoundException extends MetaDataEntityNotFoundException { - private static final long serialVersionUID = 1L; - private static SQLExceptionCode code = SQLExceptionCode.SCHEMA_NOT_FOUND; - private final String schemaName; - - public SchemaNotFoundException(String schemaName) { - super(new SQLExceptionInfo.Builder(code).setSchemaName(schemaName).build().toString(), - code.getSQLState(), code.getErrorCode()); - this.schemaName = schemaName; - } - - public String getSchemaName() { - return schemaName; - } - -} diff --git a/src/main/java/com/salesforce/phoenix/util/SchemaUtil.java b/src/main/java/com/salesforce/phoenix/util/SchemaUtil.java index 7544468a..08eaa50f 100644 --- a/src/main/java/com/salesforce/phoenix/util/SchemaUtil.java +++ b/src/main/java/com/salesforce/phoenix/util/SchemaUtil.java @@ -41,6 +41,7 @@ import static com.salesforce.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM_NAME; import static com.salesforce.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SCHEMA_AND_TABLE; import static com.salesforce.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_TABLE_NAME; +import static com.salesforce.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_TABLE_NAME_BYTES; import java.io.IOException; import java.sql.Connection; @@ -270,10 +271,13 @@ public static String getTableName(byte[] schemaName, byte[] tableName) { return Bytes.toString(getTableNameAsBytes(schemaName,tableName)); } - public static String getColumnName(String schemaName, String tableName, String familyName, String columnName) { - if ((schemaName == null || schemaName.isEmpty()) && tableName == null || tableName.isEmpty()) { + public static String getMetaDataEntityName(String schemaName, String tableName, String familyName, String columnName) { + if ((schemaName == null || schemaName.isEmpty()) && (tableName == null || tableName.isEmpty())) { return getName(familyName, columnName); } + if ((familyName == null || familyName.isEmpty()) && (columnName == null || columnName.isEmpty())) { + return getName(schemaName, tableName); + } return getName(getName(schemaName, tableName), getName(familyName, columnName)); } @@ -399,7 +403,7 @@ public static byte[] getEmptyColumnFamily(List families) { } public static boolean isMetaTable(byte[] tableName) { - return Bytes.compareTo(tableName, TYPE_TABLE_NAME) == 0; + return Bytes.compareTo(tableName, TYPE_TABLE_NAME_BYTES) == 0; } public static byte[] padChar(byte[] byteValue, Integer byteSize) { @@ -488,7 +492,7 @@ private static int estimatePartLength(int pos, List pkColumns) { public static final String UPGRADE_TO_2_1 = "UpgradeTo21"; public static boolean isUpgradeTo2Necessary(ConnectionQueryServices connServices) throws SQLException { - HTableInterface htable = connServices.getTable(PhoenixDatabaseMetaData.TYPE_TABLE_NAME); + HTableInterface htable = connServices.getTable(PhoenixDatabaseMetaData.TYPE_TABLE_NAME_BYTES); try { return (htable.getTableDescriptor().getValue(SchemaUtil.UPGRADE_TO_2_0) == null); } catch (IOException e) { @@ -598,7 +602,7 @@ public static boolean checkIfUpgradeTo2Necessary(ConnectionQueryServices connect info.remove(SchemaUtil.UPGRADE_TO_2_0); // Remove this property and ignore, since upgrade has already been done return false; } - return true; + return isUpgradeNecessary; } public static String getEscapedTableName(String schemaName, String tableName) { @@ -634,9 +638,9 @@ public static boolean columnExists(PTable table, String columnName) { } public static void updateSystemTableTo2(PhoenixConnection metaConnection, PTable table) throws SQLException { - PTable metaTable = metaConnection.getPMetaData().getTable(SchemaUtil.getTableName(PhoenixDatabaseMetaData.TYPE_SCHEMA, PhoenixDatabaseMetaData.TYPE_TABLE)); + PTable metaTable = metaConnection.getPMetaData().getTable(TYPE_TABLE_NAME); // Execute alter table statement for each column that was added if not already added - if (metaTable.getTimeStamp() < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP - 1) { + if (table.getTimeStamp() < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP - 1) { // Causes row key of system table to be upgraded if (checkIfUpgradeTo2Necessary(metaConnection.getQueryServices(), metaConnection.getURL(), metaConnection.getClientInfo())) { metaConnection.createStatement().executeQuery("select count(*) from " + PhoenixDatabaseMetaData.TYPE_SCHEMA_AND_TABLE).next(); @@ -734,12 +738,12 @@ public static void upgradeTo2(PhoenixConnection conn) throws SQLException { HTableInterface htable = null; HBaseAdmin admin = connServices.getAdmin(); try { - htable = connServices.getTable(PhoenixDatabaseMetaData.TYPE_TABLE_NAME); + htable = connServices.getTable(PhoenixDatabaseMetaData.TYPE_TABLE_NAME_BYTES); HTableDescriptor htd = new HTableDescriptor(htable.getTableDescriptor()); htd.setValue(SchemaUtil.UPGRADE_TO_2_0, Boolean.TRUE.toString()); - admin.disableTable(PhoenixDatabaseMetaData.TYPE_TABLE_NAME); - admin.modifyTable(PhoenixDatabaseMetaData.TYPE_TABLE_NAME, htd); - admin.enableTable(PhoenixDatabaseMetaData.TYPE_TABLE_NAME); + admin.disableTable(PhoenixDatabaseMetaData.TYPE_TABLE_NAME_BYTES); + admin.modifyTable(PhoenixDatabaseMetaData.TYPE_TABLE_NAME_BYTES, htd); + admin.enableTable(PhoenixDatabaseMetaData.TYPE_TABLE_NAME_BYTES); } catch (IOException e) { throw new SQLException(e); } finally { diff --git a/src/test/java/com/salesforce/phoenix/schema/SchemaUtilTest.java b/src/test/java/com/salesforce/phoenix/schema/SchemaUtilTest.java index f6f37cc0..3012759f 100644 --- a/src/test/java/com/salesforce/phoenix/schema/SchemaUtilTest.java +++ b/src/test/java/com/salesforce/phoenix/schema/SchemaUtilTest.java @@ -46,15 +46,15 @@ public void testGetTableName() { @Test public void testGetColumnName() { String columnDisplayName; - columnDisplayName = SchemaUtil.getColumnName("schemaName", "tableName", "familyName", "columnName"); + columnDisplayName = SchemaUtil.getMetaDataEntityName("schemaName", "tableName", "familyName", "columnName"); assertEquals(columnDisplayName, "schemaName.tableName.familyName.columnName"); - columnDisplayName = SchemaUtil.getColumnName(null, "tableName", "familyName", "columnName"); + columnDisplayName = SchemaUtil.getMetaDataEntityName(null, "tableName", "familyName", "columnName"); assertEquals(columnDisplayName, "tableName.familyName.columnName"); - columnDisplayName = SchemaUtil.getColumnName("schemaName", "tableName", null, "columnName"); + columnDisplayName = SchemaUtil.getMetaDataEntityName("schemaName", "tableName", null, "columnName"); assertEquals(columnDisplayName, "schemaName.tableName.columnName"); - columnDisplayName = SchemaUtil.getColumnName(null, null, "familyName", "columnName"); + columnDisplayName = SchemaUtil.getMetaDataEntityName(null, null, "familyName", "columnName"); assertEquals(columnDisplayName, "familyName.columnName"); - columnDisplayName = SchemaUtil.getColumnName(null, null, null, "columnName"); + columnDisplayName = SchemaUtil.getMetaDataEntityName(null, null, null, "columnName"); assertEquals(columnDisplayName, "columnName"); } } diff --git a/src/test/java/com/salesforce/phoenix/util/TestUtil.java b/src/test/java/com/salesforce/phoenix/util/TestUtil.java index bc02c46b..12a5ba1f 100644 --- a/src/test/java/com/salesforce/phoenix/util/TestUtil.java +++ b/src/test/java/com/salesforce/phoenix/util/TestUtil.java @@ -273,7 +273,7 @@ public static void compareTuples(Tuple res1, Tuple res2) public static void clearMetaDataCache(Connection conn) throws Throwable { PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); - HTableInterface htable = pconn.getQueryServices().getTable(PhoenixDatabaseMetaData.TYPE_TABLE_NAME); + HTableInterface htable = pconn.getQueryServices().getTable(PhoenixDatabaseMetaData.TYPE_TABLE_NAME_BYTES); htable.coprocessorExec(MetaDataProtocol.class, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, new Batch.Call() { @Override From b814a8d0ee251d0d77a4cd3192d1669053af18ac Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Mon, 16 Sep 2013 14:15:45 -0700 Subject: [PATCH 029/102] Fixing index writes from batch updates. Essentially, a batch will call the postDelete/Put hooks for each put/delete in the batch with the same WAL. Given the way we do the indexing (all the updates in the WAL), this lead to the index being updated for each Put/Delete in the batch, rather than one at a time. --- .../salesforce/hbase/index/IndexWriter.java | 6 ++-- .../com/salesforce/hbase/index/Indexer.java | 35 +++++++++++++++---- .../hbase/index/wal/IndexedKeyValue.java | 12 ++++++- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/salesforce/hbase/index/IndexWriter.java b/src/main/java/com/salesforce/hbase/index/IndexWriter.java index 9394e51d..370fade9 100644 --- a/src/main/java/com/salesforce/hbase/index/IndexWriter.java +++ b/src/main/java/com/salesforce/hbase/index/IndexWriter.java @@ -111,8 +111,10 @@ public void write(Collection> updates) for (Pair entry : toWrite) { // do the put into the index table singleMutation.add(entry.getFirst()); - LOG.info("Writing index update:" + entry.getFirst() + " to table: " - + entry.getSecond().getTableName()); + if (LOG.isDebugEnabled()) { + LOG.debug("Writing index update:" + entry.getFirst() + " to table: " + + entry.getSecond().getTableName()); + } try { HTableInterface table; diff --git a/src/main/java/com/salesforce/hbase/index/Indexer.java b/src/main/java/com/salesforce/hbase/index/Indexer.java index 53b8360e..2d962ef7 100644 --- a/src/main/java/com/salesforce/hbase/index/Indexer.java +++ b/src/main/java/com/salesforce/hbase/index/Indexer.java @@ -240,15 +240,20 @@ private void doPost(WALEdit edit, Mutation m, boolean writeToWAL) { return; } - Collection> indexUpdates = extractIndexUpdate(edit); - - // early exit - we have nothing to write, so we don't need to do anything else. NOTE: we don't - // release the WAL Rolling lock (INDEX_UPDATE_LOCK) since we never take it in doPre if there are - // no index updates. - if (indexUpdates.size() == 0) { + IndexedKeyValue ikv = getFirstIndexedKeyValue(edit); + /* + * early exit - we have nothing to write, so we don't need to do anything else. Alternatively, + * we only want to write the batch once (this hook gets called with the same WALEdit for each + * Put/Delete in a batch, which can lead to writing all the index updates for each Put/Delete). + * NOTE: we don't release the WAL Rolling lock (INDEX_UPDATE_LOCK) since we never take it in + * doPre if there are no index updates and it was already released if we completed the batch. + */ + if (ikv == null || ikv.getBatchFinished()) { return; } + Collection> indexUpdates = extractIndexUpdate(edit); + // the WAL edit is kept in memory and we already specified the factory when we created the // references originally - therefore, we just pass in a null factory here and use the ones // already specified on each reference @@ -256,6 +261,24 @@ private void doPost(WALEdit edit, Mutation m, boolean writeToWAL) { // release the lock on the index, we wrote everything properly INDEX_UPDATE_LOCK.unlock(); + // mark the batch as having been written. In the single-update case, this never gets check + // again, but in the batch case, we will check it again (see above). + ikv.markBatchFinished(); + } + + /** + * Search the {@link WALEdit} for the first {@link IndexedKeyValue} present + * @param edit {@link WALEdit} + * @return the first {@link IndexedKeyValue} in the {@link WALEdit} or null if not + * present + */ + private IndexedKeyValue getFirstIndexedKeyValue(WALEdit edit) { + for (KeyValue kv : edit.getKeyValues()) { + if (kv instanceof IndexedKeyValue) { + return (IndexedKeyValue) kv; + } + } + return null; } /** diff --git a/src/main/java/com/salesforce/hbase/index/wal/IndexedKeyValue.java b/src/main/java/com/salesforce/hbase/index/wal/IndexedKeyValue.java index 87846807..5d9d36ae 100644 --- a/src/main/java/com/salesforce/hbase/index/wal/IndexedKeyValue.java +++ b/src/main/java/com/salesforce/hbase/index/wal/IndexedKeyValue.java @@ -15,6 +15,8 @@ public class IndexedKeyValue extends KeyValue { String indexTableName; Mutation mutation; + // optimization check to ensure that batches don't get replayed to the index more than once + private boolean batchFinished = false; public IndexedKeyValue() { } @@ -43,7 +45,7 @@ public boolean matchingFamily(final byte[] family) { @Override public String toString() { - return "IndexWrite - table: " + indexTableName + ", mutation:" + mutation; + return "IndexWrite:\n\ttable: " + indexTableName + "\n\tmutation:" + mutation; } /** @@ -125,4 +127,12 @@ public void readFields(DataInput in) throws IOException { throw new IOException(e); } } + + public boolean getBatchFinished() { + return this.batchFinished; + } + + public void markBatchFinished() { + this.batchFinished = true; + } } \ No newline at end of file From b9bf69eb7e92c98ab1cad15217473d344b16b1cc Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Mon, 16 Sep 2013 14:49:01 -0700 Subject: [PATCH 030/102] Fixing locking issue on the indexer. Previously, we weren't releasing the LogRoll lock the correct number of times. Even if the batch is completed, we need to release the roll-lock for each put/delete. --- .../com/salesforce/hbase/index/Indexer.java | 39 ++++++++++++------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/salesforce/hbase/index/Indexer.java b/src/main/java/com/salesforce/hbase/index/Indexer.java index 2d962ef7..07afed89 100644 --- a/src/main/java/com/salesforce/hbase/index/Indexer.java +++ b/src/main/java/com/salesforce/hbase/index/Indexer.java @@ -239,31 +239,40 @@ private void doPost(WALEdit edit, Mutation m, boolean writeToWAL) { if(!this.builder.isEnabled(m)){ return; } - + // there is a little bit of excess here- we iterate all the non-index kvs for this check first + // and then do it again later when getting out the index updates. This should be pretty minor + // though, compared to the rest of the runtime IndexedKeyValue ikv = getFirstIndexedKeyValue(edit); /* - * early exit - we have nothing to write, so we don't need to do anything else. Alternatively, - * we only want to write the batch once (this hook gets called with the same WALEdit for each - * Put/Delete in a batch, which can lead to writing all the index updates for each Put/Delete). - * NOTE: we don't release the WAL Rolling lock (INDEX_UPDATE_LOCK) since we never take it in - * doPre if there are no index updates and it was already released if we completed the batch. + * early exit - we have nothing to write, so we don't need to do anything else. NOTE: we don't + * release the WAL Rolling lock (INDEX_UPDATE_LOCK) since we never take it in doPre if there are + * no index updates. */ - if (ikv == null || ikv.getBatchFinished()) { + if (ikv == null) { return; } - Collection> indexUpdates = extractIndexUpdate(edit); + /* + * only write the update if we haven't already seen this batch. We only want to write the batch + * once (this hook gets called with the same WALEdit for each Put/Delete in a batch, which can + * lead to writing all the index updates for each Put/Delete). + */ + if (!ikv.getBatchFinished()) { + Collection> indexUpdates = extractIndexUpdate(edit); - // the WAL edit is kept in memory and we already specified the factory when we created the - // references originally - therefore, we just pass in a null factory here and use the ones - // already specified on each reference - writer.writeAndKillYourselfOnFailure(indexUpdates); + // the WAL edit is kept in memory and we already specified the factory when we created the + // references originally - therefore, we just pass in a null factory here and use the ones + // already specified on each reference + writer.writeAndKillYourselfOnFailure(indexUpdates); + + // mark the batch as having been written. In the single-update case, this never gets check + // again, but in the batch case, we will check it again (see above). + ikv.markBatchFinished(); + } // release the lock on the index, we wrote everything properly + // we took the lock for each Put/Delete, so we have to release it a matching number of times INDEX_UPDATE_LOCK.unlock(); - // mark the batch as having been written. In the single-update case, this never gets check - // again, but in the batch case, we will check it again (see above). - ikv.markBatchFinished(); } /** From 73940e37e80e748f8f783f8006db76d9a9d3fa9d Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Mon, 16 Sep 2013 14:34:17 -0700 Subject: [PATCH 031/102] Batching index updates by table on write Previously, each index mutation was sent as part of a single batch to the target table, leading to a lot of opened and closed writes. Now we do all the updates to a table at once, leading to much better batching efficiency, etc. --- .../index/CannotReachIndexException.java | 8 +++- .../salesforce/hbase/index/IndexWriter.java | 45 ++++++++++--------- .../index/table/HTableInterfaceReference.java | 10 +++++ 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/src/main/java/com/salesforce/hbase/index/CannotReachIndexException.java b/src/main/java/com/salesforce/hbase/index/CannotReachIndexException.java index 60bb8dd3..40c490d8 100644 --- a/src/main/java/com/salesforce/hbase/index/CannotReachIndexException.java +++ b/src/main/java/com/salesforce/hbase/index/CannotReachIndexException.java @@ -27,6 +27,8 @@ ******************************************************************************/ package com.salesforce.hbase.index; +import java.util.List; + import org.apache.hadoop.hbase.client.Mutation; /** @@ -35,7 +37,9 @@ @SuppressWarnings("serial") public class CannotReachIndexException extends Exception { - public CannotReachIndexException(String targetTableName, Mutation m, Exception cause) { - super("Cannot reach index table " + targetTableName + " to update index for edit: " + m, cause); + public CannotReachIndexException(String targetTableName, List mutations, Exception cause) { + super( + "Cannot reach index table " + targetTableName + " to update index for edit: " + mutations, + cause); } } diff --git a/src/main/java/com/salesforce/hbase/index/IndexWriter.java b/src/main/java/com/salesforce/hbase/index/IndexWriter.java index 370fade9..d9fb88de 100644 --- a/src/main/java/com/salesforce/hbase/index/IndexWriter.java +++ b/src/main/java/com/salesforce/hbase/index/IndexWriter.java @@ -34,6 +34,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import org.apache.commons.logging.Log; @@ -46,6 +47,8 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; import com.salesforce.hbase.index.table.HTableFactory; import com.salesforce.hbase.index.table.HTableInterfaceReference; @@ -79,8 +82,8 @@ public IndexWriter(String sourceInfo, Abortable abortable, HTableFactory factory * Just write the index update portions of of the edit, if it is an {@link IndexedWALEdit}. If it * is not passed an {@link IndexedWALEdit}, any further actions are ignored. *

- * Internally, uses {@link #wri to make the write and if is receives a { - * @link CannotReachIndexException}, it attempts to move ( + * Internally, uses {@link #write(Collection)} to make the write and if is receives a + * {@link CannotReachIndexException}, it attempts to move ( * {@link HBaseAdmin#unassign(byte[], boolean)}) the region and then failing that calls * {@link System#exit(int)} to kill the server. */ @@ -101,37 +104,36 @@ public void writeAndKillYourselfOnFailure(Collection> ind */ public void write(Collection> updates) throws CannotReachIndexException { - // conver the strings to htableinterfaces to which we can talk - Collection> toWrite = + // convert the strings to htableinterfaces to which we can talk and group by TABLE + Multimap toWrite = resolveTableReferences(factory, updates); // write each mutation, as a part of a batch, to its respective table - List singleMutation = new ArrayList(1); + List mutations; Set tables = new HashSet(); - for (Pair entry : toWrite) { - // do the put into the index table - singleMutation.add(entry.getFirst()); + for (Entry> entry : toWrite.asMap().entrySet()) { + // get the mutations for each table. We leak the implementation here a little bit to save + // doing a complete copy over of all the index update for each table. + mutations = (List) entry.getValue(); if (LOG.isDebugEnabled()) { - LOG.debug("Writing index update:" + entry.getFirst() + " to table: " - + entry.getSecond().getTableName()); + LOG.debug("Writing index update:" + mutations + " to table: " + + entry.getKey().getTableName()); } try { HTableInterface table; if (factory == null) { - table = entry.getSecond().getTable(); + table = entry.getKey().getTable(); } else { - table = entry.getSecond().getTable(factory); + table = entry.getKey().getTable(factory); } - // do the update - table.batch(singleMutation); + table.batch(mutations); tables.add(table); } catch (IOException e) { - throw new CannotReachIndexException(entry.getSecond().getTableName(), entry.getFirst(), e); + throw new CannotReachIndexException(entry.getKey().getTableName(), mutations, e); } catch (InterruptedException e) { - throw new CannotReachIndexException(entry.getSecond().getTableName(), entry.getFirst(), e); + throw new CannotReachIndexException(entry.getKey().getTableName(), mutations, e); } - singleMutation.clear(); } // go through each reference and close the connection // we can't do this earlier as we may reuse table references between different index entries, @@ -169,11 +171,10 @@ private void killYourself(Throwable cause) { * @param indexUpdates from the index builder * @return pairs that can then be written by an {@link IndexWriter}. */ - public static Collection> resolveTableReferences( + public static Multimap resolveTableReferences( HTableFactory factory, Collection> indexUpdates) { - - Collection> updates = - new ArrayList>(indexUpdates.size()); + Multimap updates = + ArrayListMultimap. create(); Map tables = new HashMap(updates.size()); for (Pair entry : indexUpdates) { @@ -183,7 +184,7 @@ public static Collection> resolveTableR table = new HTableInterfaceReference(entry.getSecond(), factory); tables.put(tableName, table); } - updates.add(new Pair(entry.getFirst(), table)); + updates.put(table, entry.getFirst()); } return updates; diff --git a/src/main/java/com/salesforce/hbase/index/table/HTableInterfaceReference.java b/src/main/java/com/salesforce/hbase/index/table/HTableInterfaceReference.java index 42637df4..82399069 100644 --- a/src/main/java/com/salesforce/hbase/index/table/HTableInterfaceReference.java +++ b/src/main/java/com/salesforce/hbase/index/table/HTableInterfaceReference.java @@ -82,4 +82,14 @@ public void readFields(DataInput in) throws IOException { public void write(DataOutput out) throws IOException { out.writeUTF(this.tableName); } + + @Override + public int hashCode() { + return this.tableName.hashCode(); + } + + @Override + public boolean equals(Object o) { + return o == null ? false : this.hashCode() == o.hashCode(); + } } \ No newline at end of file From 57bf58a8d555ce34d64cea2fd624e6ef82941a3b Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Mon, 16 Sep 2013 15:55:06 -0700 Subject: [PATCH 032/102] Switching internal memstore to no use MemSlab. Using the memslab causes an the KV to be array copied to the internal slab, an unnecessary expenditure b/c we can just reuse the existing SLAB (and backing array) for the KVs that we get passed. --- .../salesforce/hbase/index/covered/LocalTableState.java | 7 ++++++- .../apache/hadoop/hbase/regionserver/ExposedMemStore.java | 8 ++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java b/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java index 370730a0..b4bcdff5 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java +++ b/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java @@ -36,6 +36,7 @@ import java.util.Map; import java.util.Set; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Result; @@ -78,7 +79,11 @@ public LocalTableState(RegionCoprocessorEnvironment environment, LocalHBaseState this.env = environment; this.table = table; this.update = update; - this.memstore = new ExposedMemStore(this.env.getConfiguration(), KeyValue.COMPARATOR); + // ensure that the memstore just uses the KV references, rather than copying them into the + // memstore + Configuration conf = new Configuration(environment.getConfiguration()); + ExposedMemStore.disableMemSLAB(conf); + this.memstore = new ExposedMemStore(conf, KeyValue.COMPARATOR); this.scannerBuilder = new ScannerBuilder(memstore, update); } diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ExposedMemStore.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ExposedMemStore.java index 30ec89f4..b97be293 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/ExposedMemStore.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ExposedMemStore.java @@ -81,4 +81,12 @@ public long heapSizeChange(KeyValue kv, boolean notpresent) { return super.heapSizeChange(kv, notpresent); } + /** + * Disable the memstore MemSLAB in the given configuration. + * @param conf to update + */ + public static void disableMemSLAB(Configuration conf) { + conf.setBoolean(MemStore.USEMSLAB_KEY, false); + } + } From 90b5646c67e66bc6aaa32e4bfe0f7ddd170656a3 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Mon, 16 Sep 2013 16:05:19 -0700 Subject: [PATCH 033/102] Use JDBC standard names for table types (Issue #432) --- .../phoenix/expression/ExpressionType.java | 4 +- .../phoenix/jdbc/PhoenixDatabaseMetaData.java | 45 ++++++++++--- .../salesforce/phoenix/schema/PTableType.java | 64 ++++++++++++++----- .../end2end/BaseConnectedQueryTest.java | 33 ++++++++-- .../phoenix/end2end/ProductMetricsTest.java | 20 ++++++ .../end2end/QueryDatabaseMetaDataTest.java | 20 +++--- .../phoenix/end2end/QueryExecTest.java | 37 +++++++++-- .../end2end/index/IndexMetadataTest.java | 25 ++++++-- 8 files changed, 196 insertions(+), 52 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/expression/ExpressionType.java b/src/main/java/com/salesforce/phoenix/expression/ExpressionType.java index ce70f94c..b3c4f702 100644 --- a/src/main/java/com/salesforce/phoenix/expression/ExpressionType.java +++ b/src/main/java/com/salesforce/phoenix/expression/ExpressionType.java @@ -47,6 +47,7 @@ import com.salesforce.phoenix.expression.function.RegexpSubstrFunction; import com.salesforce.phoenix.expression.function.ReverseFunction; import com.salesforce.phoenix.expression.function.RoundFunction; +import com.salesforce.phoenix.expression.function.SqlTableType; import com.salesforce.phoenix.expression.function.SqlTypeNameFunction; import com.salesforce.phoenix.expression.function.StddevPopFunction; import com.salesforce.phoenix.expression.function.StddevSampFunction; @@ -123,7 +124,8 @@ public enum ExpressionType { DoubleSubtractExpression(DoubleSubtractExpression.class), DoubleMultiplyExpression(DoubleMultiplyExpression.class), DoubleDivideExpression(DoubleDivideExpression.class), - MD5Function(MD5Function.class); + MD5Function(MD5Function.class), + SqlTableType(SqlTableType.class); ExpressionType(Class clazz) { this.clazz = clazz; diff --git a/src/main/java/com/salesforce/phoenix/jdbc/PhoenixDatabaseMetaData.java b/src/main/java/com/salesforce/phoenix/jdbc/PhoenixDatabaseMetaData.java index c436ac4a..6fdc73cc 100644 --- a/src/main/java/com/salesforce/phoenix/jdbc/PhoenixDatabaseMetaData.java +++ b/src/main/java/com/salesforce/phoenix/jdbc/PhoenixDatabaseMetaData.java @@ -52,6 +52,7 @@ import com.salesforce.phoenix.exception.SQLExceptionInfo; import com.salesforce.phoenix.expression.BaseTerminalExpression; import com.salesforce.phoenix.expression.RowKeyColumnExpression; +import com.salesforce.phoenix.expression.function.SqlTableType; import com.salesforce.phoenix.expression.function.SqlTypeNameFunction; import com.salesforce.phoenix.iterate.DelegateResultIterator; import com.salesforce.phoenix.iterate.MaterializedResultIterator; @@ -79,6 +80,21 @@ * JDBC DatabaseMetaData implementation of Phoenix reflecting read-only nature of driver. * Supported metadata methods include: * {@link #getTables(String, String, String, String[])} + * {@link #getColumns(String, String, String, String)} + * {@link #getTableTypes()} + * {@link #getPrimaryKeys(String, String, String)} + * {@link #getIndexInfo(String, String, String, boolean, boolean)} + * {@link #getSchemas()} + * {@link #getSchemas(String, String)} + * {@link #getDatabaseMajorVersion()} + * {@link #getDatabaseMinorVersion()} + * {@link #getClientInfoProperties()} + * {@link #getConnection()} + * {@link #getDatabaseProductName()} + * {@link #getDatabaseProductVersion()} + * {@link #getDefaultTransactionIsolation()} + * {@link #getDriverName()} + * {@link #getDriverVersion()} * Other ResultSet methods return an empty result set. * * @author jtaylor @@ -749,7 +765,6 @@ public ResultSet getTablePrivileges(String catalog, String schemaPattern, String return emptyResultSet; } - private static final Integer TABLE_TYPE_MAX_LENGTH = 1; private static final PDatum TABLE_TYPE_DATUM = new PDatum() { @Override public boolean isNullable() { @@ -757,11 +772,11 @@ public boolean isNullable() { } @Override public PDataType getDataType() { - return PDataType.CHAR; + return PDataType.VARCHAR; } @Override public Integer getByteSize() { - return TABLE_TYPE_MAX_LENGTH; + return null; } @Override public Integer getMaxLength() { @@ -784,7 +799,7 @@ public ColumnModifier getColumnModifier() { private static final Collection TABLE_TYPE_TUPLES = Lists.newArrayListWithExpectedSize(PTableType.values().length); static { for (PTableType tableType : PTableType.values()) { - TABLE_TYPE_TUPLES.add(new SingleKeyValueTuple(KeyValueUtil.newKeyValue(PDataType.CHAR.toBytes(tableType.getSerializedValue()), TABLE_FAMILY_BYTES, TABLE_TYPE_BYTES, MetaDataProtocol.MIN_TABLE_TIMESTAMP, ByteUtil.EMPTY_BYTE_ARRAY))); + TABLE_TYPE_TUPLES.add(new SingleKeyValueTuple(KeyValueUtil.newKeyValue(tableType.getValue().getBytes(), TABLE_FAMILY_BYTES, TABLE_TYPE_BYTES, MetaDataProtocol.MIN_TABLE_TIMESTAMP, ByteUtil.EMPTY_BYTE_ARRAY))); } } private static final Scanner TABLE_TYPE_SCANNER = new WrappedScanner(new MaterializedResultIterator(TABLE_TYPE_TUPLES),TABLE_TYPE_ROW_PROJECTOR); @@ -814,7 +829,7 @@ public ResultSet getTables(String catalog, String schemaPattern, String tableNam TABLE_CAT_NAME + "," + // no catalog for tables TABLE_SCHEM_NAME + "," + TABLE_NAME_NAME + " ," + - TABLE_TYPE_NAME + "," + + SqlTableType.NAME + "(" + TABLE_TYPE_NAME + ") AS " + TABLE_TYPE_NAME + "," + REMARKS_NAME + " ," + TYPE_NAME + "," + SELF_REFERENCING_COL_NAME_NAME + "," + @@ -832,11 +847,21 @@ public ResultSet getTables(String catalog, String schemaPattern, String tableNam } if (types != null && types.length > 0) { buf.append(" and " + TABLE_TYPE_NAME + " IN ("); - for (String type : types) { - buf.append('\''); - buf.append(type); - buf.append('\''); - buf.append(','); + // For b/w compat as table types changed in 2.2.0 TODO remove in 3.0 + if (types[0].length() == 1) { + for (String type : types) { + buf.append('\''); + buf.append(type); + buf.append('\''); + buf.append(','); + } + } else { + for (String type : types) { + buf.append('\''); + buf.append(PTableType.fromValue(type).getSerializedValue()); + buf.append('\''); + buf.append(','); + } } buf.setCharAt(buf.length()-1, ')'); } diff --git a/src/main/java/com/salesforce/phoenix/schema/PTableType.java b/src/main/java/com/salesforce/phoenix/schema/PTableType.java index d7086312..7579149a 100644 --- a/src/main/java/com/salesforce/phoenix/schema/PTableType.java +++ b/src/main/java/com/salesforce/phoenix/schema/PTableType.java @@ -27,25 +27,41 @@ ******************************************************************************/ package com.salesforce.phoenix.schema; +import java.util.Map; + +import com.google.common.collect.Maps; + public enum PTableType { - SYSTEM("s"), - USER("u"), - VIEW("v"), - INDEX("i"); + SYSTEM("s", "SYSTEM TABLE"), + USER("u", "TABLE"), + VIEW("v", "VIEW"), + INDEX("i", "INDEX"); + private final PName value; private final String serializedValue; - private PTableType(String serializedValue) { + private PTableType(String serializedValue, String value) { this.serializedValue = serializedValue; + this.value = PNameFactory.newName(value); } public String getSerializedValue() { return serializedValue; } - private static final PTableType[] FROM_VALUE; - private static final int FROM_VALUE_OFFSET; + public PName getValue() { + return value; + } + + @Override + public String toString() { + return value.getString(); + } + + private static final PTableType[] FROM_SERIALIZED_VALUE; + private static final int FROM_SERIALIZED_VALUE_OFFSET; + private static final Map FROM_VALUE = Maps.newHashMapWithExpectedSize(PTableType.values().length); static { int minChar = Integer.MAX_VALUE; @@ -59,27 +75,43 @@ public String getSerializedValue() { maxChar = c; } } - FROM_VALUE_OFFSET = minChar; - FROM_VALUE = new PTableType[maxChar - minChar + 1]; + FROM_SERIALIZED_VALUE_OFFSET = minChar; + FROM_SERIALIZED_VALUE = new PTableType[maxChar - minChar + 1]; for (PTableType type : PTableType.values()) { - FROM_VALUE[type.getSerializedValue().charAt(0) - minChar] = type; + FROM_SERIALIZED_VALUE[type.getSerializedValue().charAt(0) - minChar] = type; + } + } + + static { + for (PTableType type : PTableType.values()) { + if (FROM_VALUE.put(type.getValue().getString(),type) != null) { + throw new IllegalStateException("Duplicate PTableType value of " + type.getValue().getString() + " is not allowed"); + } + } + } + + public static PTableType fromValue(String value) { + PTableType type = FROM_VALUE.get(value); + if (type == null) { + throw new IllegalArgumentException("Unable to PTableType enum for value of '" + value + "'"); } + return type; } public static PTableType fromSerializedValue(String serializedValue) { if (serializedValue.length() == 1) { - int i = serializedValue.charAt(0) - FROM_VALUE_OFFSET; - if (i >= 0 && i < FROM_VALUE.length && FROM_VALUE[i] != null) { - return FROM_VALUE[i]; + int i = serializedValue.charAt(0) - FROM_SERIALIZED_VALUE_OFFSET; + if (i >= 0 && i < FROM_SERIALIZED_VALUE.length && FROM_SERIALIZED_VALUE[i] != null) { + return FROM_SERIALIZED_VALUE[i]; } } throw new IllegalArgumentException("Unable to PTableType enum for serialized value of '" + serializedValue + "'"); } public static PTableType fromSerializedValue(byte serializedByte) { - int i = serializedByte - FROM_VALUE_OFFSET; - if (i >= 0 && i < FROM_VALUE.length && FROM_VALUE[i] != null) { - return FROM_VALUE[i]; + int i = serializedByte - FROM_SERIALIZED_VALUE_OFFSET; + if (i >= 0 && i < FROM_SERIALIZED_VALUE.length && FROM_SERIALIZED_VALUE[i] != null) { + return FROM_SERIALIZED_VALUE[i]; } throw new IllegalArgumentException("Unable to PTableType enum for serialized value of '" + (char)serializedByte + "'"); } diff --git a/src/test/java/com/salesforce/phoenix/end2end/BaseConnectedQueryTest.java b/src/test/java/com/salesforce/phoenix/end2end/BaseConnectedQueryTest.java index 4546cc90..b190166d 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/BaseConnectedQueryTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/BaseConnectedQueryTest.java @@ -27,10 +27,31 @@ ******************************************************************************/ package com.salesforce.phoenix.end2end; -import static com.salesforce.phoenix.util.TestUtil.*; +import static com.salesforce.phoenix.util.TestUtil.ATABLE_NAME; +import static com.salesforce.phoenix.util.TestUtil.A_VALUE; +import static com.salesforce.phoenix.util.TestUtil.B_VALUE; +import static com.salesforce.phoenix.util.TestUtil.C_VALUE; +import static com.salesforce.phoenix.util.TestUtil.E_VALUE; +import static com.salesforce.phoenix.util.TestUtil.MILLIS_IN_DAY; +import static com.salesforce.phoenix.util.TestUtil.PHOENIX_JDBC_URL; +import static com.salesforce.phoenix.util.TestUtil.ROW1; +import static com.salesforce.phoenix.util.TestUtil.ROW2; +import static com.salesforce.phoenix.util.TestUtil.ROW3; +import static com.salesforce.phoenix.util.TestUtil.ROW4; +import static com.salesforce.phoenix.util.TestUtil.ROW5; +import static com.salesforce.phoenix.util.TestUtil.ROW6; +import static com.salesforce.phoenix.util.TestUtil.ROW7; +import static com.salesforce.phoenix.util.TestUtil.ROW8; +import static com.salesforce.phoenix.util.TestUtil.ROW9; import java.math.BigDecimal; -import java.sql.*; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.Date; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Types; import java.util.Properties; import org.apache.hadoop.hbase.HConstants; @@ -40,7 +61,9 @@ import com.salesforce.phoenix.jdbc.PhoenixDatabaseMetaData; import com.salesforce.phoenix.query.BaseTest; import com.salesforce.phoenix.schema.PTableType; -import com.salesforce.phoenix.util.*; +import com.salesforce.phoenix.util.PhoenixRuntime; +import com.salesforce.phoenix.util.SchemaUtil; +import com.salesforce.phoenix.util.TestUtil; /** @@ -76,12 +99,12 @@ protected static void deletePriorTables(long ts) throws Exception { Connection conn = DriverManager.getConnection(PHOENIX_JDBC_URL, props); try { DatabaseMetaData dbmd = conn.getMetaData(); - ResultSet rs = dbmd.getTables(null, null, null, new String[] {PTableType.USER.getSerializedValue(), PTableType.VIEW.getSerializedValue()}); + ResultSet rs = dbmd.getTables(null, null, null, new String[] {PTableType.USER.toString(), PTableType.VIEW.toString()}); while (rs.next()) { String fullTableName = SchemaUtil.getTableName( rs.getString(PhoenixDatabaseMetaData.TABLE_SCHEM_NAME), rs.getString(PhoenixDatabaseMetaData.TABLE_NAME_NAME)); - conn.createStatement().executeUpdate("DROP " + (PTableType.fromSerializedValue(rs.getString(PhoenixDatabaseMetaData.TABLE_TYPE_NAME)) == PTableType.VIEW ? "VIEW " : "TABLE ") + fullTableName); + conn.createStatement().executeUpdate("DROP " + rs.getString(PhoenixDatabaseMetaData.TABLE_TYPE_NAME) + " " + fullTableName); } } finally { conn.close(); diff --git a/src/test/java/com/salesforce/phoenix/end2end/ProductMetricsTest.java b/src/test/java/com/salesforce/phoenix/end2end/ProductMetricsTest.java index 2f90efe9..c23a1081 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/ProductMetricsTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/ProductMetricsTest.java @@ -340,6 +340,26 @@ public void testDateRangeAggregation() throws Exception { } } + @Test + public void testTableAliasSameAsTableName() throws Exception { + long ts = nextTimestamp(); + String tenantId = getOrganizationId(); + String query = "SELECT sum(transactions) FROM PRODUCT_METRICS PRODUCT_METRICS"; + String url = PHOENIX_JDBC_URL + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + (ts + 5); // Run query at timestamp 5 + Properties props = new Properties(TEST_PROPERTIES); + Connection conn = DriverManager.getConnection(url, props); + try { + initTableValues(tenantId, getSplits(tenantId), ts); + PreparedStatement statement = conn.prepareStatement(query); + ResultSet rs = statement.executeQuery(); + assertTrue(rs.next()); + assertEquals(2100, rs.getLong(1)); + assertFalse(rs.next()); + } finally { + conn.close(); + } + } + @Test public void testPartiallyEvaluableAnd() throws Exception { long ts = nextTimestamp(); diff --git a/src/test/java/com/salesforce/phoenix/end2end/QueryDatabaseMetaDataTest.java b/src/test/java/com/salesforce/phoenix/end2end/QueryDatabaseMetaDataTest.java index 838f6c92..98d8ed5e 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/QueryDatabaseMetaDataTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/QueryDatabaseMetaDataTest.java @@ -103,34 +103,34 @@ public void testTableMetadataScan() throws SQLException { ResultSet rs = dbmd.getTables(null, aSchemaName, aTableName, null); assertTrue(rs.next()); assertEquals(rs.getString("TABLE_NAME"),aTableName); - assertEquals(PTableType.USER.getSerializedValue(), rs.getString("TABLE_TYPE")); + assertEquals(PTableType.USER.toString(), rs.getString("TABLE_TYPE")); assertEquals(rs.getString(3),aTableName); - assertEquals(PTableType.USER.getSerializedValue(), rs.getString(4)); + assertEquals(PTableType.USER.toString(), rs.getString(4)); assertFalse(rs.next()); rs = dbmd.getTables(null, null, null, null); assertTrue(rs.next()); assertEquals(rs.getString("TABLE_SCHEM"),TYPE_SCHEMA); assertEquals(rs.getString("TABLE_NAME"),TYPE_TABLE); - assertEquals(PTableType.SYSTEM.getSerializedValue(), rs.getString("TABLE_TYPE")); + assertEquals(PTableType.SYSTEM.toString(), rs.getString("TABLE_TYPE")); assertTrue(rs.next()); assertEquals(rs.getString("TABLE_SCHEM"),null); assertEquals(rs.getString("TABLE_NAME"),ATABLE_NAME); - assertEquals(PTableType.USER.getSerializedValue(), rs.getString("TABLE_TYPE")); + assertEquals(PTableType.USER.toString(), rs.getString("TABLE_TYPE")); assertTrue(rs.next()); assertEquals(rs.getString("TABLE_SCHEM"),null); assertEquals(rs.getString("TABLE_NAME"),STABLE_NAME); - assertEquals(PTableType.USER.getSerializedValue(), rs.getString("TABLE_TYPE")); + assertEquals(PTableType.USER.toString(), rs.getString("TABLE_TYPE")); assertTrue(rs.next()); assertEquals(CUSTOM_ENTITY_DATA_SCHEMA_NAME, rs.getString("TABLE_SCHEM")); assertEquals(CUSTOM_ENTITY_DATA_NAME, rs.getString("TABLE_NAME")); - assertEquals(PTableType.USER.getSerializedValue(), rs.getString("TABLE_TYPE")); + assertEquals(PTableType.USER.toString(), rs.getString("TABLE_TYPE")); rs = dbmd.getTables(null, CUSTOM_ENTITY_DATA_SCHEMA_NAME, CUSTOM_ENTITY_DATA_NAME, null); assertTrue(rs.next()); assertEquals(rs.getString("TABLE_SCHEM"),CUSTOM_ENTITY_DATA_SCHEMA_NAME); assertEquals(rs.getString("TABLE_NAME"),CUSTOM_ENTITY_DATA_NAME); - assertEquals(PTableType.USER.getSerializedValue(), rs.getString("TABLE_TYPE")); + assertEquals(PTableType.USER.toString(), rs.getString("TABLE_TYPE")); assertFalse(rs.next()); try { @@ -141,15 +141,15 @@ public void testTableMetadataScan() throws SQLException { } assertFalse(rs.next()); - rs = dbmd.getTables(null, "", "_TABLE", new String[] {PTableType.USER.getSerializedValue()}); + rs = dbmd.getTables(null, "", "_TABLE", new String[] {PTableType.USER.toString()}); assertTrue(rs.next()); assertEquals(rs.getString("TABLE_SCHEM"),null); assertEquals(rs.getString("TABLE_NAME"),ATABLE_NAME); - assertEquals(PTableType.USER.getSerializedValue(), rs.getString("TABLE_TYPE")); + assertEquals(PTableType.USER.toString(), rs.getString("TABLE_TYPE")); assertTrue(rs.next()); assertEquals(rs.getString("TABLE_SCHEM"),null); assertEquals(rs.getString("TABLE_NAME"),STABLE_NAME); - assertEquals(PTableType.USER.getSerializedValue(), rs.getString("TABLE_TYPE")); + assertEquals(PTableType.USER.toString(), rs.getString("TABLE_TYPE")); assertFalse(rs.next()); } diff --git a/src/test/java/com/salesforce/phoenix/end2end/QueryExecTest.java b/src/test/java/com/salesforce/phoenix/end2end/QueryExecTest.java index 990325b0..313741fb 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/QueryExecTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/QueryExecTest.java @@ -27,13 +27,42 @@ ******************************************************************************/ package com.salesforce.phoenix.end2end; -import static com.salesforce.phoenix.util.TestUtil.*; -import static org.junit.Assert.*; +import static com.salesforce.phoenix.util.TestUtil.A_VALUE; +import static com.salesforce.phoenix.util.TestUtil.B_VALUE; +import static com.salesforce.phoenix.util.TestUtil.C_VALUE; +import static com.salesforce.phoenix.util.TestUtil.E_VALUE; +import static com.salesforce.phoenix.util.TestUtil.MILLIS_IN_DAY; +import static com.salesforce.phoenix.util.TestUtil.PHOENIX_JDBC_URL; +import static com.salesforce.phoenix.util.TestUtil.ROW1; +import static com.salesforce.phoenix.util.TestUtil.ROW2; +import static com.salesforce.phoenix.util.TestUtil.ROW3; +import static com.salesforce.phoenix.util.TestUtil.ROW4; +import static com.salesforce.phoenix.util.TestUtil.ROW5; +import static com.salesforce.phoenix.util.TestUtil.ROW6; +import static com.salesforce.phoenix.util.TestUtil.ROW7; +import static com.salesforce.phoenix.util.TestUtil.ROW8; +import static com.salesforce.phoenix.util.TestUtil.ROW9; +import static com.salesforce.phoenix.util.TestUtil.TEST_PROPERTIES; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.math.BigDecimal; -import java.sql.*; +import java.sql.Connection; import java.sql.Date; -import java.util.*; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.HashSet; +import java.util.List; +import java.util.Properties; +import java.util.Set; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; diff --git a/src/test/java/com/salesforce/phoenix/end2end/index/IndexMetadataTest.java b/src/test/java/com/salesforce/phoenix/end2end/index/IndexMetadataTest.java index 3b5149bd..e509d996 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/index/IndexMetadataTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/index/IndexMetadataTest.java @@ -1,9 +1,20 @@ package com.salesforce.phoenix.end2end.index; -import static com.salesforce.phoenix.util.TestUtil.*; -import static org.junit.Assert.*; +import static com.salesforce.phoenix.util.TestUtil.INDEX_DATA_SCHEMA; +import static com.salesforce.phoenix.util.TestUtil.INDEX_DATA_TABLE; +import static com.salesforce.phoenix.util.TestUtil.TEST_PROPERTIES; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; -import java.sql.*; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; import java.util.Properties; import org.junit.Test; @@ -11,7 +22,9 @@ import com.salesforce.phoenix.end2end.BaseHBaseManagedTimeTest; import com.salesforce.phoenix.exception.SQLExceptionCode; import com.salesforce.phoenix.query.QueryConstants; -import com.salesforce.phoenix.schema.*; +import com.salesforce.phoenix.schema.AmbiguousColumnException; +import com.salesforce.phoenix.schema.PIndexState; +import com.salesforce.phoenix.schema.PTableType; import com.salesforce.phoenix.util.StringUtil; import com.salesforce.phoenix.util.TestUtil; @@ -94,7 +107,7 @@ public void testIndexCreation() throws Exception { ddl = "ALTER INDEX IDX ON " + INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + INDEX_DATA_TABLE + " DISABLE"; conn.createStatement().execute(ddl); // Verify the metadata for index is correct. - rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(INDEX_DATA_SCHEMA), "IDX", new String[] {PTableType.INDEX.getSerializedValue()}); + rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(INDEX_DATA_SCHEMA), "IDX", new String[] {PTableType.INDEX.toString()}); assertTrue(rs.next()); assertEquals("IDX", rs.getString(3)); assertEquals(PIndexState.INACTIVE.getSerializedValue(), rs.getString("INDEX_STATE")); @@ -197,7 +210,7 @@ public void testIndexDefinitionWithNullableFixedWidthColInPK() throws Exception ddl = "ALTER INDEX IDX ON " + INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + INDEX_DATA_TABLE + " DISABLE"; conn.createStatement().execute(ddl); // Verify the metadata for index is correct. - rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(INDEX_DATA_SCHEMA), "IDX", new String[] {PTableType.INDEX.getSerializedValue()}); + rs = conn.getMetaData().getTables(null, StringUtil.escapeLike(INDEX_DATA_SCHEMA), "IDX", new String[] {PTableType.INDEX.toString()}); assertTrue(rs.next()); assertEquals("IDX", rs.getString(3)); assertEquals(PIndexState.INACTIVE.getSerializedValue(), rs.getString("INDEX_STATE")); From de218295380c73fe6f5ea743be2fea5047e1956f Mon Sep 17 00:00:00 2001 From: James Taylor Date: Mon, 16 Sep 2013 16:48:41 -0700 Subject: [PATCH 034/102] Add missing java file --- .../expression/function/SqlTableType.java | 88 +++++++++++++++++++ .../phoenix/index/PhoenixIndexBuilder.java | 7 ++ .../phoenix/index/PhoenixIndexCodec.java | 2 - 3 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/salesforce/phoenix/expression/function/SqlTableType.java diff --git a/src/main/java/com/salesforce/phoenix/expression/function/SqlTableType.java b/src/main/java/com/salesforce/phoenix/expression/function/SqlTableType.java new file mode 100644 index 00000000..e7fba277 --- /dev/null +++ b/src/main/java/com/salesforce/phoenix/expression/function/SqlTableType.java @@ -0,0 +1,88 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.phoenix.expression.function; + +import java.sql.SQLException; +import java.util.List; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; + +import com.salesforce.phoenix.expression.Expression; +import com.salesforce.phoenix.parse.FunctionParseNode.Argument; +import com.salesforce.phoenix.parse.FunctionParseNode.BuiltInFunction; +import com.salesforce.phoenix.schema.PDataType; +import com.salesforce.phoenix.schema.PTableType; +import com.salesforce.phoenix.schema.tuple.Tuple; + + +/** + * + * Function used to get the SQL table type name from the serialized table type. + * Usage: + * SqlTableType('v') will return 'VIEW' based on + * {@link java.sql.DatabaseMetaData#getTableTypes()} + * + * @author jtaylor + * @since 2.2 + */ +@BuiltInFunction(name=SqlTableType.NAME, args= { + @Argument(allowedTypes=PDataType.CHAR)} ) +public class SqlTableType extends ScalarFunction { + public static final String NAME = "SqlTableType"; + + public SqlTableType() { + } + + public SqlTableType(List children) throws SQLException { + super(children); + } + + @Override + public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { + Expression child = children.get(0); + if (!child.evaluate(tuple, ptr)) { + return false; + } + if (ptr.getLength() == 0) { + return true; + } + PTableType tableType = PTableType.fromSerializedValue(ptr.get()[ptr.getOffset()]); + ptr.set(tableType.getValue().getBytes()); + return true; + } + + @Override + public PDataType getDataType() { + return PDataType.VARCHAR; + } + + @Override + public String getName() { + return NAME; + } +} diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java index c0adcdc2..b0e7cf12 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java @@ -28,6 +28,8 @@ package com.salesforce.phoenix.index; import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; +import org.apache.hadoop.hbase.util.Pair; import com.salesforce.hbase.index.covered.CoveredColumnsIndexBuilder; @@ -36,6 +38,11 @@ */ public class PhoenixIndexBuilder extends CoveredColumnsIndexBuilder { + @Override + public void batchStarted(MiniBatchOperationInProgress> miniBatchOp) { + // TODO + } + @Override public boolean isEnabled(Mutation m) { // ask the codec to see if we should even attempt indexing diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java index 96829d6a..5919b73f 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java @@ -160,8 +160,6 @@ public Iterable getIndexDeletes(TableState state) throws IOExceptio @Override public boolean isEnabled(Mutation m) { - // TODO cache these maintainers so we don't need to rediscover them later (e.g. when building - // the index update) List maintainers = getIndexMaintainers(m.getAttributesMap()); return maintainers.size() > 0; } From ffb657186122a0771229e09a33318a0e474b4e95 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Tue, 17 Sep 2013 07:49:06 -0700 Subject: [PATCH 035/102] Fix for parser issue for NOT with parenthesis (Issue #433) --- src/main/antlr3/PhoenixSQL.g | 10 +-- .../salesforce/hbase/index/IndexWriter.java | 1 - .../phoenix/compile/ExpressionCompiler.java | 69 +++++++++++++++++-- .../phoenix/compile/HavingCompiler.java | 26 ++++++- .../phoenix/compile/WhereCompiler.java | 30 ++++++-- .../index/IndexMetaDataCacheClient.java | 14 +--- .../phoenix/index/PhoenixIndexBuilder.java | 2 +- .../com/salesforce/phoenix/util/ScanUtil.java | 23 ++++++- .../phoenix/compile/QueryCompileTest.java | 4 +- .../phoenix/parse/QueryParserTest.java | 52 +++++++++++--- 10 files changed, 187 insertions(+), 44 deletions(-) diff --git a/src/main/antlr3/PhoenixSQL.g b/src/main/antlr3/PhoenixSQL.g index 6bafdc6c..27738ddf 100644 --- a/src/main/antlr3/PhoenixSQL.g +++ b/src/main/antlr3/PhoenixSQL.g @@ -655,13 +655,12 @@ condition_and returns [ParseNode ret] // NOT or parenthesis condition_not returns [ParseNode ret] - : ( boolean_expr ) => e=boolean_expr { $ret = e; } - | NOT e=boolean_expr { $ret = factory.not(e); } - | LPAREN e=condition RPAREN { $ret = e; } + : (NOT? boolean_expr ) => n=NOT? e=boolean_expr { $ret = n == null ? e : factory.not(e); } + | n=NOT? LPAREN e=condition RPAREN { $ret = n == null ? e : factory.not(e); } ; boolean_expr returns [ParseNode ret] - : (l=expression ((EQ r=expression {$ret = factory.equal(l,r); } ) + : l=expression ((EQ r=expression {$ret = factory.equal(l,r); } ) | ((NOEQ1 | NOEQ2) r=expression {$ret = factory.notEqual(l,r); } ) | (LT r=expression {$ret = factory.lt(l,r); } ) | (GT r=expression {$ret = factory.gt(l,r); } ) @@ -675,7 +674,8 @@ boolean_expr returns [ParseNode ret] | (LPAREN r=select_expression RPAREN {$ret = factory.in(l,r,n!=null);} ) | (v=values {List il = new ArrayList(v.size() + 1); il.add(l); il.addAll(v); $ret = factory.inList(il,n!=null);}) ))) - )))) + )) + | { $ret = l; } ) ; bind_expression returns [BindParseNode ret] diff --git a/src/main/java/com/salesforce/hbase/index/IndexWriter.java b/src/main/java/com/salesforce/hbase/index/IndexWriter.java index d9fb88de..7fdcd7b6 100644 --- a/src/main/java/com/salesforce/hbase/index/IndexWriter.java +++ b/src/main/java/com/salesforce/hbase/index/IndexWriter.java @@ -28,7 +28,6 @@ package com.salesforce.hbase.index; import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; diff --git a/src/main/java/com/salesforce/phoenix/compile/ExpressionCompiler.java b/src/main/java/com/salesforce/phoenix/compile/ExpressionCompiler.java index 807a1808..fa2ff89b 100644 --- a/src/main/java/com/salesforce/phoenix/compile/ExpressionCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/ExpressionCompiler.java @@ -30,7 +30,10 @@ import java.math.BigDecimal; import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; @@ -38,11 +41,63 @@ import com.salesforce.phoenix.compile.GroupByCompiler.GroupBy; import com.salesforce.phoenix.exception.SQLExceptionCode; import com.salesforce.phoenix.exception.SQLExceptionInfo; -import com.salesforce.phoenix.expression.*; +import com.salesforce.phoenix.expression.AndExpression; +import com.salesforce.phoenix.expression.CaseExpression; +import com.salesforce.phoenix.expression.CoerceExpression; +import com.salesforce.phoenix.expression.ComparisonExpression; +import com.salesforce.phoenix.expression.DateAddExpression; +import com.salesforce.phoenix.expression.DateSubtractExpression; +import com.salesforce.phoenix.expression.DecimalAddExpression; +import com.salesforce.phoenix.expression.DecimalDivideExpression; +import com.salesforce.phoenix.expression.DecimalMultiplyExpression; +import com.salesforce.phoenix.expression.DecimalSubtractExpression; +import com.salesforce.phoenix.expression.DoubleAddExpression; +import com.salesforce.phoenix.expression.DoubleDivideExpression; +import com.salesforce.phoenix.expression.DoubleMultiplyExpression; +import com.salesforce.phoenix.expression.DoubleSubtractExpression; +import com.salesforce.phoenix.expression.Expression; +import com.salesforce.phoenix.expression.InListExpression; +import com.salesforce.phoenix.expression.IsNullExpression; +import com.salesforce.phoenix.expression.LikeExpression; +import com.salesforce.phoenix.expression.LiteralExpression; +import com.salesforce.phoenix.expression.LongAddExpression; +import com.salesforce.phoenix.expression.LongDivideExpression; +import com.salesforce.phoenix.expression.LongMultiplyExpression; +import com.salesforce.phoenix.expression.LongSubtractExpression; +import com.salesforce.phoenix.expression.NotExpression; +import com.salesforce.phoenix.expression.OrExpression; +import com.salesforce.phoenix.expression.RowKeyColumnExpression; +import com.salesforce.phoenix.expression.StringConcatExpression; import com.salesforce.phoenix.expression.function.FunctionExpression; -import com.salesforce.phoenix.parse.*; +import com.salesforce.phoenix.parse.AddParseNode; +import com.salesforce.phoenix.parse.AndParseNode; +import com.salesforce.phoenix.parse.ArithmeticParseNode; +import com.salesforce.phoenix.parse.BindParseNode; +import com.salesforce.phoenix.parse.CaseParseNode; +import com.salesforce.phoenix.parse.CastParseNode; +import com.salesforce.phoenix.parse.ColumnParseNode; +import com.salesforce.phoenix.parse.ComparisonParseNode; +import com.salesforce.phoenix.parse.DivideParseNode; +import com.salesforce.phoenix.parse.FunctionParseNode; import com.salesforce.phoenix.parse.FunctionParseNode.BuiltInFunctionInfo; -import com.salesforce.phoenix.schema.*; +import com.salesforce.phoenix.parse.InListParseNode; +import com.salesforce.phoenix.parse.IsNullParseNode; +import com.salesforce.phoenix.parse.LikeParseNode; +import com.salesforce.phoenix.parse.LiteralParseNode; +import com.salesforce.phoenix.parse.MultiplyParseNode; +import com.salesforce.phoenix.parse.NotParseNode; +import com.salesforce.phoenix.parse.OrParseNode; +import com.salesforce.phoenix.parse.ParseNode; +import com.salesforce.phoenix.parse.StringConcatParseNode; +import com.salesforce.phoenix.parse.SubtractParseNode; +import com.salesforce.phoenix.parse.UnsupportedAllParseNodeVisitor; +import com.salesforce.phoenix.schema.ColumnModifier; +import com.salesforce.phoenix.schema.ColumnRef; +import com.salesforce.phoenix.schema.DelegateDatum; +import com.salesforce.phoenix.schema.PDataType; +import com.salesforce.phoenix.schema.PDatum; +import com.salesforce.phoenix.schema.RowKeyValueAccessor; +import com.salesforce.phoenix.schema.TypeMismatchException; import com.salesforce.phoenix.util.ByteUtil; import com.salesforce.phoenix.util.SchemaUtil; @@ -230,6 +285,9 @@ public Expression visitLeave(AndParseNode node, List children) throw Iterator iterator = children.iterator(); while (iterator.hasNext()) { Expression child = iterator.next(); + if (child.getDataType() != PDataType.BOOLEAN) { + throw new TypeMismatchException(PDataType.BOOLEAN, child.getDataType(), child.toString()); + } if (child == LiteralExpression.FALSE_EXPRESSION) { return child; } @@ -255,6 +313,9 @@ private Expression orExpression(List children) throws SQLException { Iterator iterator = children.iterator(); while (iterator.hasNext()) { Expression child = iterator.next(); + if (child.getDataType() != PDataType.BOOLEAN) { + throw new TypeMismatchException(PDataType.BOOLEAN, child.getDataType(), child.toString()); + } if (child == LiteralExpression.FALSE_EXPRESSION) { iterator.remove(); } diff --git a/src/main/java/com/salesforce/phoenix/compile/HavingCompiler.java b/src/main/java/com/salesforce/phoenix/compile/HavingCompiler.java index cf081084..3baf975f 100644 --- a/src/main/java/com/salesforce/phoenix/compile/HavingCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/HavingCompiler.java @@ -28,15 +28,34 @@ package com.salesforce.phoenix.compile; import java.sql.SQLException; -import java.util.*; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; import com.salesforce.phoenix.compile.GroupByCompiler.GroupBy; import com.salesforce.phoenix.exception.SQLExceptionCode; import com.salesforce.phoenix.exception.SQLExceptionInfo; import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.expression.LiteralExpression; -import com.salesforce.phoenix.parse.*; +import com.salesforce.phoenix.parse.AddParseNode; +import com.salesforce.phoenix.parse.AndParseNode; +import com.salesforce.phoenix.parse.BetweenParseNode; +import com.salesforce.phoenix.parse.CaseParseNode; +import com.salesforce.phoenix.parse.ColumnParseNode; +import com.salesforce.phoenix.parse.ComparisonParseNode; +import com.salesforce.phoenix.parse.DivideParseNode; +import com.salesforce.phoenix.parse.FunctionParseNode; +import com.salesforce.phoenix.parse.IsNullParseNode; +import com.salesforce.phoenix.parse.MultiplyParseNode; +import com.salesforce.phoenix.parse.OrParseNode; +import com.salesforce.phoenix.parse.ParseNode; +import com.salesforce.phoenix.parse.SelectStatement; +import com.salesforce.phoenix.parse.SelectStatementRewriter; +import com.salesforce.phoenix.parse.SubtractParseNode; +import com.salesforce.phoenix.parse.TraverseNoParseNodeVisitor; import com.salesforce.phoenix.schema.ColumnRef; +import com.salesforce.phoenix.schema.PDataType; +import com.salesforce.phoenix.schema.TypeMismatchException; public class HavingCompiler { @@ -51,6 +70,9 @@ public static Expression compile(StatementContext context, SelectStatement state } ExpressionCompiler expressionBuilder = new ExpressionCompiler(context, groupBy); Expression expression = having.accept(expressionBuilder); + if (expression.getDataType() != PDataType.BOOLEAN) { + throw new TypeMismatchException(PDataType.BOOLEAN, expression.getDataType(), expression.toString()); + } if (LiteralExpression.FALSE_EXPRESSION == expression) { context.setScanRanges(ScanRanges.NOTHING); return null; diff --git a/src/main/java/com/salesforce/phoenix/compile/WhereCompiler.java b/src/main/java/com/salesforce/phoenix/compile/WhereCompiler.java index 4cef4a22..bcbdde36 100644 --- a/src/main/java/com/salesforce/phoenix/compile/WhereCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/WhereCompiler.java @@ -39,13 +39,30 @@ import com.google.common.collect.Sets; import com.salesforce.phoenix.exception.SQLExceptionCode; import com.salesforce.phoenix.exception.SQLExceptionInfo; -import com.salesforce.phoenix.expression.*; +import com.salesforce.phoenix.expression.Expression; +import com.salesforce.phoenix.expression.KeyValueColumnExpression; +import com.salesforce.phoenix.expression.LiteralExpression; import com.salesforce.phoenix.expression.visitor.KeyValueExpressionVisitor; -import com.salesforce.phoenix.filter.*; -import com.salesforce.phoenix.parse.*; +import com.salesforce.phoenix.filter.MultiCFCQKeyValueComparisonFilter; +import com.salesforce.phoenix.filter.MultiCQKeyValueComparisonFilter; +import com.salesforce.phoenix.filter.RowKeyComparisonFilter; +import com.salesforce.phoenix.filter.SingleCFCQKeyValueComparisonFilter; +import com.salesforce.phoenix.filter.SingleCQKeyValueComparisonFilter; +import com.salesforce.phoenix.parse.ColumnParseNode; +import com.salesforce.phoenix.parse.FilterableStatement; import com.salesforce.phoenix.parse.HintNode.Hint; -import com.salesforce.phoenix.schema.*; -import com.salesforce.phoenix.util.*; +import com.salesforce.phoenix.parse.ParseNode; +import com.salesforce.phoenix.parse.ParseNodeFactory; +import com.salesforce.phoenix.schema.AmbiguousColumnException; +import com.salesforce.phoenix.schema.ColumnNotFoundException; +import com.salesforce.phoenix.schema.ColumnRef; +import com.salesforce.phoenix.schema.PDataType; +import com.salesforce.phoenix.schema.PTable; +import com.salesforce.phoenix.schema.PTableType; +import com.salesforce.phoenix.schema.TypeMismatchException; +import com.salesforce.phoenix.util.ByteUtil; +import com.salesforce.phoenix.util.ScanUtil; +import com.salesforce.phoenix.util.SchemaUtil; /** @@ -90,6 +107,9 @@ public static Expression compileWhereClause(StatementContext context, Filterable if (whereCompiler.isAggregate()) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.AGGREGATE_IN_WHERE).build().buildException(); } + if (expression.getDataType() != PDataType.BOOLEAN) { + throw new TypeMismatchException(PDataType.BOOLEAN, expression.getDataType(), expression.toString()); + } expression = WhereOptimizer.pushKeyExpressionsToScan(context, statement, expression, extractedNodes); setScanFilter(context, statement, expression, whereCompiler.disambiguateWithFamily); diff --git a/src/main/java/com/salesforce/phoenix/index/IndexMetaDataCacheClient.java b/src/main/java/com/salesforce/phoenix/index/IndexMetaDataCacheClient.java index 40f60c64..e0bc9176 100644 --- a/src/main/java/com/salesforce/phoenix/index/IndexMetaDataCacheClient.java +++ b/src/main/java/com/salesforce/phoenix/index/IndexMetaDataCacheClient.java @@ -1,22 +1,17 @@ package com.salesforce.phoenix.index; import java.sql.SQLException; -import java.util.Collections; import java.util.List; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import com.google.common.collect.Lists; import com.salesforce.phoenix.cache.ServerCacheClient; import com.salesforce.phoenix.cache.ServerCacheClient.ServerCache; -import com.salesforce.phoenix.compile.ScanRanges; import com.salesforce.phoenix.jdbc.PhoenixConnection; import com.salesforce.phoenix.join.MaxServerCacheSizeExceededException; -import com.salesforce.phoenix.query.KeyRange; -import com.salesforce.phoenix.schema.PDataType; -import com.salesforce.phoenix.schema.SaltingUtil; import com.salesforce.phoenix.schema.TableRef; +import com.salesforce.phoenix.util.ScanUtil; public class IndexMetaDataCacheClient { private static final int USE_CACHE_THRESHOLD = 10; @@ -56,15 +51,10 @@ public static boolean useIndexMetadataCache(List mutations, int indexM * size */ public ServerCache addIndexMetadataCache(List mutations, ImmutableBytesWritable ptr) throws SQLException { - List keys = Lists.newArrayListWithExpectedSize(mutations.size()); - for (Mutation m : mutations) { - keys.add(PDataType.VARBINARY.getKeyRange(m.getRow())); - } - ScanRanges keyRanges = ScanRanges.create(Collections.singletonList(keys), SaltingUtil.VAR_BINARY_SCHEMA); /** * Serialize and compress hashCacheTable */ - return serverCache.addServerCache(keyRanges, ptr, new IndexMetaDataCacheFactory()); + return serverCache.addServerCache(ScanUtil.newScanRanges(mutations), ptr, new IndexMetaDataCacheFactory()); } } diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java index b0e7cf12..ad2898c6 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java @@ -40,7 +40,7 @@ public class PhoenixIndexBuilder extends CoveredColumnsIndexBuilder { @Override public void batchStarted(MiniBatchOperationInProgress> miniBatchOp) { - // TODO + // TODO: use skip scan here } @Override diff --git a/src/main/java/com/salesforce/phoenix/util/ScanUtil.java b/src/main/java/com/salesforce/phoenix/util/ScanUtil.java index 3d9ae24a..156b6257 100644 --- a/src/main/java/com/salesforce/phoenix/util/ScanUtil.java +++ b/src/main/java/com/salesforce/phoenix/util/ScanUtil.java @@ -28,19 +28,29 @@ package com.salesforce.phoenix.util; import java.io.IOException; -import java.util.*; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; +import com.google.common.collect.Lists; +import com.salesforce.phoenix.compile.ScanRanges; import com.salesforce.phoenix.coprocessor.MetaDataProtocol; import com.salesforce.phoenix.filter.SkipScanFilter; -import com.salesforce.phoenix.query.*; +import com.salesforce.phoenix.query.KeyRange; import com.salesforce.phoenix.query.KeyRange.Bound; +import com.salesforce.phoenix.query.QueryConstants; +import com.salesforce.phoenix.schema.PDataType; import com.salesforce.phoenix.schema.RowKeySchema; +import com.salesforce.phoenix.schema.SaltingUtil; /** @@ -382,4 +392,13 @@ public static int searchClosestKeyRangeWithUpperHigherThanPtr(List slo return ++mid; } } + + public static ScanRanges newScanRanges(List mutations) throws SQLException { + List keys = Lists.newArrayListWithExpectedSize(mutations.size()); + for (Mutation m : mutations) { + keys.add(PDataType.VARBINARY.getKeyRange(m.getRow())); + } + ScanRanges keyRanges = ScanRanges.create(Collections.singletonList(keys), SaltingUtil.VAR_BINARY_SCHEMA); + return keyRanges; + } } diff --git a/src/test/java/com/salesforce/phoenix/compile/QueryCompileTest.java b/src/test/java/com/salesforce/phoenix/compile/QueryCompileTest.java index 7cbccf8d..191fe5ab 100644 --- a/src/test/java/com/salesforce/phoenix/compile/QueryCompileTest.java +++ b/src/test/java/com/salesforce/phoenix/compile/QueryCompileTest.java @@ -320,7 +320,7 @@ public void testTypeMismatchInCase() throws Exception { public void testNonBooleanWhereExpression() throws Exception { try { // Select non agg column in aggregate query - String query = "SELECT a_integer FROM atable WHERE organization_id=? and CASE WHEN a_integer <= 2 THEN 'foo' WHEN a_integer = 3 THEN 2 WHEN a_integer <= 5 THEN 5 ELSE 5 END"; + String query = "SELECT a_integer FROM atable WHERE organization_id=? and CASE WHEN a_integer <= 2 THEN 'foo' WHEN a_integer = 3 THEN 'bar' WHEN a_integer <= 5 THEN 'bas' ELSE 'blah' END"; Properties props = new Properties(TestUtil.TEST_PROPERTIES); Connection conn = DriverManager.getConnection(getUrl(), props); try { @@ -332,7 +332,7 @@ public void testNonBooleanWhereExpression() throws Exception { conn.close(); } } catch (SQLException e) { - assertTrue(e.getMessage().contains("ERROR 601 (42P00): Syntax error. Encountered \"CASE\" at line 1, column 58.")); + assertTrue(e.getMessage().contains("ERROR 203 (22005): Type mismatch. BOOLEAN and VARCHAR for CASE WHEN A_INTEGER <= 2 THEN 'foo'WHEN A_INTEGER = 3 THEN 'bar'WHEN A_INTEGER <= 5 THEN 'bas' ELSE 'blah' END")); } } diff --git a/src/test/java/com/salesforce/phoenix/parse/QueryParserTest.java b/src/test/java/com/salesforce/phoenix/parse/QueryParserTest.java index 18148a3c..c87a6e6d 100644 --- a/src/test/java/com/salesforce/phoenix/parse/QueryParserTest.java +++ b/src/test/java/com/salesforce/phoenix/parse/QueryParserTest.java @@ -27,7 +27,9 @@ ******************************************************************************/ package com.salesforce.phoenix.parse; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.StringReader; import java.sql.SQLException; @@ -390,15 +392,6 @@ public void testParsingStatementWithMissingToken() throws Exception { } catch (SQLException e) { assertTrue(e.getMessage(), e.getMessage().contains("ERROR 603 (42P00): Syntax error. Mismatched input. Expecting \"FROM\", got \"where\" at line 2, column 1.")); } - try { - SQLParser parser = new SQLParser(new StringReader( - "select a from b\n" + - "where d\n")); - parser.parseStatement(); - fail("Should have caught exception."); - } catch (SQLException e) { - assertTrue(e.getMessage(), e.getMessage().contains("ERROR 601 (42P00): Syntax error. Encountered \"d\" at line 2, column 7.")); - } } @Test @@ -526,4 +519,43 @@ public void testPercentileQuery2() throws Exception { "select PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY mark ASC) from core.custom_index_value ind")); parser.parseStatement(); } + + @Test + public void testSingleTopLevelNot() throws Exception { + SQLParser parser = new SQLParser( + new StringReader( + "select * from t where not c = 5")); + parser.parseStatement(); + } + + @Test + public void testTopLevelNot() throws Exception { + SQLParser parser = new SQLParser( + new StringReader( + "select * from t where not c")); + parser.parseStatement(); + } + + @Test + public void testHavingWithNot() throws Exception { + SQLParser parser = new SQLParser( + new StringReader( + "select\n" + + "\"WEB_STAT_ALIAS\".\"DOMAIN\" as \"c0\"\n" + + "from \"WEB_STAT\" \"WEB_STAT_ALIAS\"\n" + + "group by \"WEB_STAT_ALIAS\".\"DOMAIN\" having\n" + + "(\n" + + "(\n" + + "NOT\n" + + "(\n" + + "(sum(\"WEB_STAT_ALIAS\".\"ACTIVE_VISITOR\") is null)\n" + + ")\n" + + "OR NOT((sum(\"WEB_STAT_ALIAS\".\"ACTIVE_VISITOR\") is null))\n" + + ")\n" + + "OR NOT((sum(\"WEB_STAT_ALIAS\".\"ACTIVE_VISITOR\") is null))\n" + + ")\n" + + "order by CASE WHEN \"WEB_STAT_ALIAS\".\"DOMAIN\" IS NULL THEN 1 ELSE 0 END,\n" + + "\"WEB_STAT_ALIAS\".\"DOMAIN\" ASC")); + parser.parseStatement(); + } } From 8fa3cf47b7ae6ce87d6bea6ec5f4ec2f098544d7 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Tue, 17 Sep 2013 13:40:02 -0700 Subject: [PATCH 036/102] Use skip scan to prime block cache for secondary index maintenance --- .../hbase/index/builder/BaseIndexBuilder.java | 2 +- .../hbase/index/builder/IndexBuilder.java | 3 +- .../phoenix/index/PhoenixIndexBuilder.java | 54 +++++++++++++++++-- .../phoenix/index/PhoenixIndexCodec.java | 4 ++ 4 files changed, 58 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/salesforce/hbase/index/builder/BaseIndexBuilder.java b/src/main/java/com/salesforce/hbase/index/builder/BaseIndexBuilder.java index 07181650..68cf1f7b 100644 --- a/src/main/java/com/salesforce/hbase/index/builder/BaseIndexBuilder.java +++ b/src/main/java/com/salesforce/hbase/index/builder/BaseIndexBuilder.java @@ -57,7 +57,7 @@ public void setup(RegionCoprocessorEnvironment conf) throws IOException { } @Override - public void batchStarted(MiniBatchOperationInProgress> miniBatchOp) { + public void batchStarted(MiniBatchOperationInProgress> miniBatchOp) throws IOException { // noop } diff --git a/src/main/java/com/salesforce/hbase/index/builder/IndexBuilder.java b/src/main/java/com/salesforce/hbase/index/builder/IndexBuilder.java index ae98da92..f821ae75 100644 --- a/src/main/java/com/salesforce/hbase/index/builder/IndexBuilder.java +++ b/src/main/java/com/salesforce/hbase/index/builder/IndexBuilder.java @@ -106,8 +106,9 @@ public Collection> getIndexUpdateForFilteredRows( * after the {@link #getIndexUpdate} methods. Therefore, you will likely need an attribute * on your {@link Put}/{@link Delete} to indicate it is a batch operation. * @param miniBatchOp the full batch operation to be written + * @throws IOException */ - public void batchStarted(MiniBatchOperationInProgress> miniBatchOp); + public void batchStarted(MiniBatchOperationInProgress> miniBatchOp) throws IOException; /** * This allows the codec to dynamically change whether or not indexing should take place for a diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java index ad2898c6..3aa4f302 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java @@ -27,11 +27,25 @@ ******************************************************************************/ package com.salesforce.phoenix.index; +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; +import org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl; +import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.hadoop.hbase.util.Pair; +import com.google.common.collect.Lists; import com.salesforce.hbase.index.covered.CoveredColumnsIndexBuilder; +import com.salesforce.phoenix.compile.ScanRanges; +import com.salesforce.phoenix.query.KeyRange; +import com.salesforce.phoenix.schema.PDataType; +import com.salesforce.phoenix.schema.SaltingUtil; /** * Index builder for covered-columns index that ties into phoenix for faster use. @@ -39,10 +53,44 @@ public class PhoenixIndexBuilder extends CoveredColumnsIndexBuilder { @Override - public void batchStarted(MiniBatchOperationInProgress> miniBatchOp) { - // TODO: use skip scan here + public void batchStarted(MiniBatchOperationInProgress> miniBatchOp) throws IOException { + // The entire purpose of this method impl is to get the existing rows for the + // table rows being indexed into the block cache, as the index maintenance code + // does a point scan per row + List keys = Lists.newArrayListWithExpectedSize(miniBatchOp.size()); + for (int i = 0; i < miniBatchOp.size(); i++) { + Mutation m = miniBatchOp.getOperation(i).getFirst(); + keys.add(PDataType.VARBINARY.getKeyRange(m.getRow())); + } + ScanRanges scanRanges = ScanRanges.create(Collections.singletonList(keys), SaltingUtil.VAR_BINARY_SCHEMA); + Scan scan = new Scan(); + // TODO: when the index maintenance code starts projecting only what it needs in its scan + // we should do the same here. Since it reads the entire row, projecting here wouldn't be + // good, as then it wouldn't find everything it needs in the block cache. + scan.setFilter(scanRanges.getSkipScanFilter()); + HRegion region = this.env.getRegion(); + RegionScanner scanner = region.getScanner(scan); + // Run through the scanner using internal nextRaw method + MultiVersionConsistencyControl.setThreadReadPoint(scanner.getMvccReadPoint()); + region.startRegionOperation(); + try { + boolean hasMore; + do { + List results = Lists.newArrayList(); + // Results are potentially returned even when the return value of s.next is false + // since this is an indication of whether or not there are more values after the + // ones returned + hasMore = scanner.nextRaw(results, null) && !scanner.isFilterDone(); + } while (hasMore); + } finally { + try { + scanner.close(); + } finally { + region.closeRegionOperation(); + } + } } - + @Override public boolean isEnabled(Mutation m) { // ask the codec to see if we should even attempt indexing diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java index 5919b73f..00077b45 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java @@ -61,6 +61,10 @@ public void initialize(RegionCoprocessorEnvironment env) { this.conf = env.getConfiguration(); } + public List getIndexMaintainers() { + return indexMaintainers; + } + private List getIndexMaintainers(TableState state) { return getIndexMaintainers(state.getUpdateAttributes()); } From d0fb1692584efb8d605236f8e0b846d36db397b6 Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Tue, 17 Sep 2013 12:09:26 -0700 Subject: [PATCH 037/102] Getting index's local table state directly HRegion. Used to go through the HTable interface, which right now uses the full HTable to get access. We know we are just accessing a region on the local server, so going through the full serialization and deserialization is excessive. --- .../salesforce/hbase/index/covered/Batch.java | 3 +-- .../hbase/index/covered/LocalTableState.java | 8 ++++---- .../hbase/index/covered/data/LocalTable.java | 20 +++++++++++++------ .../hbase/regionserver/ExposedMemStore.java | 13 ++++++++++++ 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/salesforce/hbase/index/covered/Batch.java b/src/main/java/com/salesforce/hbase/index/covered/Batch.java index 923afc80..14647e43 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/Batch.java +++ b/src/main/java/com/salesforce/hbase/index/covered/Batch.java @@ -28,7 +28,6 @@ package com.salesforce.hbase.index.covered; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import org.apache.hadoop.hbase.KeyValue; @@ -65,7 +64,7 @@ public long getTimestamp() { return this.timestamp; } - public Collection getKvs() { + public List getKvs() { return this.batch; } } \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java b/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java index b4bcdff5..5e7757a9 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java +++ b/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java @@ -72,7 +72,7 @@ public class LocalTableState implements TableState { private Set trackedColumns = new HashSet(); private boolean initialized; private ScannerBuilder scannerBuilder; - private Collection kvs = new ArrayList(); + private List kvs = new ArrayList(); private List hints; public LocalTableState(RegionCoprocessorEnvironment environment, LocalHBaseState table, Mutation update) { @@ -83,7 +83,7 @@ public LocalTableState(RegionCoprocessorEnvironment environment, LocalHBaseState // memstore Configuration conf = new Configuration(environment.getConfiguration()); ExposedMemStore.disableMemSLAB(conf); - this.memstore = new ExposedMemStore(conf, KeyValue.COMPARATOR); + this.memstore = new ExposedMemStore(conf, ExposedMemStore.IGNORE_MEMSTORE_TS_COMPARATOR); this.scannerBuilder = new ScannerBuilder(memstore, update); } @@ -92,13 +92,13 @@ public void addPendingUpdates(KeyValue... kvs) { addPendingUpdates(Arrays.asList(kvs)); } - public void addPendingUpdates(Collection kvs) { + public void addPendingUpdates(List kvs) { if(kvs == null) return; setPendingUpdates(kvs); addUpdate(kvs); } - private void addUpdate(Collection list) { + private void addUpdate(List list) { if (list == null) return; for (KeyValue kv : list) { this.memstore.add(kv); diff --git a/src/main/java/com/salesforce/hbase/index/covered/data/LocalTable.java b/src/main/java/com/salesforce/hbase/index/covered/data/LocalTable.java index 653c6e53..05584c64 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/data/LocalTable.java +++ b/src/main/java/com/salesforce/hbase/index/covered/data/LocalTable.java @@ -28,13 +28,18 @@ package com.salesforce.hbase.index.covered.data; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.RegionScanner; /** * Wrapper around a lazily instantiated, local HTable. @@ -61,12 +66,15 @@ public Result getCurrentRowState(Mutation m) throws IOException { Scan s = new Scan(row, row); s.setRaw(true); s.setMaxVersions(); - ResultScanner results = getLocalTable().getScanner(s); - Result r = results.next(); - assert results.next() == null : "Got more than one result when scanning" - + " a single row in the primary table!"; - results.close(); - return r == null ? new Result() : r; + HRegion region = this.env.getRegion(); + RegionScanner scanner = region.getScanner(s); + List kvs = new ArrayList(); + boolean more = scanner.next(kvs); + assert !more : "Got more than one result when scanning" + " a single row in the primary table!"; + + Result r = new Result(kvs); + scanner.close(); + return r; } /** diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ExposedMemStore.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ExposedMemStore.java index b97be293..1f099e79 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/ExposedMemStore.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ExposedMemStore.java @@ -8,6 +8,8 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KVComparator; +import com.google.common.primitives.Longs; + /** * A {@link MemStore} that exposes all the package-protected methods. *

@@ -17,6 +19,17 @@ */ public class ExposedMemStore extends MemStore { + public static KVComparator IGNORE_MEMSTORE_TS_COMPARATOR = new KVComparator() { + + public int compare(final KeyValue left, final KeyValue right) { + int ret = + getRawComparator().compare(left.getBuffer(), left.getOffset() + KeyValue.ROW_OFFSET, + left.getKeyLength(), right.getBuffer(), right.getOffset() + KeyValue.ROW_OFFSET, + right.getKeyLength()); + return ret; + } + }; + public ExposedMemStore(Configuration conf, KVComparator comparator) { super(conf, comparator); } From 71f807281ac7bd12e69928cbdf5c10afbbd6fcf5 Mon Sep 17 00:00:00 2001 From: mujtabachohan Date: Tue, 17 Sep 2013 16:30:45 -0700 Subject: [PATCH 038/102] Added logo --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a892705a..3d4ba505 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -

Phoenix: A SQL skin over HBase
-'We put the SQL back in NoSQL'

+![logo](http://forcedotcom.github.com/phoenix/images/logo.jpg) + Phoenix is a SQL skin over HBase, delivered as a client-embedded JDBC driver, powering the HBase use cases at Salesforce.com. Phoenix targets low-latency queries (milliseconds), as opposed to batch operation via map/reduce. To see what's supported, go to our [language reference guide](http://forcedotcom.github.com/phoenix/), read more on our [wiki](https://github.com/forcedotcom/phoenix/wiki), and download it [here](https://github.com/forcedotcom/phoenix/wiki/Download). ## Mission Become the standard means of accessing HBase data through a well-defined, industry standard API. From c9c29fbd45a1aee764df441859750ed11a95815d Mon Sep 17 00:00:00 2001 From: James Taylor Date: Wed, 18 Sep 2013 07:59:53 -0700 Subject: [PATCH 039/102] Modified unit test to include alias/table name reference for column, and fix related bug --- .../phoenix/compile/FromCompiler.java | 23 +++++++++++-------- .../phoenix/end2end/ProductMetricsTest.java | 2 +- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/compile/FromCompiler.java b/src/main/java/com/salesforce/phoenix/compile/FromCompiler.java index ff32c4f8..5dc984df 100644 --- a/src/main/java/com/salesforce/phoenix/compile/FromCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/FromCompiler.java @@ -103,16 +103,13 @@ public static ColumnResolver getResolver(SelectStatement statement, PhoenixConne throws SQLException { List fromNodes = statement.getFrom(); if (fromNodes.size() > 1) { throw new SQLFeatureNotSupportedException("Joins not supported"); } - MultiTableColumnResolver visitor = new MultiTableColumnResolver(connection); - for (TableNode node : fromNodes) { - node.accept(visitor); - } + SingleTableColumnResolver visitor = new SingleTableColumnResolver(connection, (NamedTableNode)fromNodes.get(0), false); return visitor; } public static ColumnResolver getResolver(SingleTableSQLStatement statement, PhoenixConnection connection, List dyn_columns) throws SQLException { - SingleTableColumnResolver visitor = new SingleTableColumnResolver(connection, statement.getTable()); + SingleTableColumnResolver visitor = new SingleTableColumnResolver(connection, statement.getTable(), true); return visitor; } @@ -122,10 +119,12 @@ public static ColumnResolver getResolver(SingleTableSQLStatement statement, Phoe } private static class SingleTableColumnResolver extends BaseColumnResolver { - private final List tableRefs; + private final List tableRefs; + private final String alias; - public SingleTableColumnResolver(PhoenixConnection connection, NamedTableNode table) throws SQLException { + public SingleTableColumnResolver(PhoenixConnection connection, NamedTableNode table, boolean updateCacheOnlyIfAutoCommit) throws SQLException { super(connection); + alias = table.getAlias(); TableName tableNameNode = table.getName(); String schemaName = tableNameNode.getSchemaName(); String tableName = tableNameNode.getTableName(); @@ -135,7 +134,7 @@ public SingleTableColumnResolver(PhoenixConnection connection, NamedTableNode ta boolean retry = true; while (true) { try { - if (connection.getAutoCommit()) { + if (!updateCacheOnlyIfAutoCommit || connection.getAutoCommit()) { timeStamp = Math.abs(client.updateCache(schemaName, tableName)); } String fullTableName = SchemaUtil.getTableName(schemaName, tableName); @@ -170,7 +169,10 @@ public List getTables() { public ColumnRef resolveColumn(String schemaName, String tableName, String colName) throws SQLException { TableRef tableRef = tableRefs.get(0); - PColumn column = tableName == null ? + if (schemaName != null && !schemaName.equals(alias)) { + throw new ColumnNotFoundException(schemaName, tableName, null, colName); + } + PColumn column = tableName == null || (schemaName == null && tableName.equals(alias)) ? tableRef.getTable().getColumn(colName) : tableRef.getTable().getColumnFamily(tableName).getColumn(colName); return new ColumnRef(tableRef, column.getPosition()); @@ -212,7 +214,8 @@ protected PTable addDynamicColumns(List dynColumns, PTable theTable) } } - private static class MultiTableColumnResolver extends BaseColumnResolver implements TableNodeVisitor { + // TODO: unused, but should be used for joins - make private once used + public static class MultiTableColumnResolver extends BaseColumnResolver implements TableNodeVisitor { private final ListMultimap tableMap; private final List tables; diff --git a/src/test/java/com/salesforce/phoenix/end2end/ProductMetricsTest.java b/src/test/java/com/salesforce/phoenix/end2end/ProductMetricsTest.java index c23a1081..113bf708 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/ProductMetricsTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/ProductMetricsTest.java @@ -344,7 +344,7 @@ public void testDateRangeAggregation() throws Exception { public void testTableAliasSameAsTableName() throws Exception { long ts = nextTimestamp(); String tenantId = getOrganizationId(); - String query = "SELECT sum(transactions) FROM PRODUCT_METRICS PRODUCT_METRICS"; + String query = "SELECT sum(PRODUCT_METRICS.transactions) FROM PRODUCT_METRICS PRODUCT_METRICS"; String url = PHOENIX_JDBC_URL + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + (ts + 5); // Run query at timestamp 5 Properties props = new Properties(TEST_PROPERTIES); Connection conn = DriverManager.getConnection(url, props); From f70b0e8bd0c1cb4cd278470defc05b23ede96040 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Wed, 18 Sep 2013 08:11:10 -0700 Subject: [PATCH 040/102] Project only what's necessary into scan, better encapsulate index maintenance --- .../phoenix/index/IndexMaintainer.java | 75 ++++++++++++++++--- .../phoenix/index/PhoenixIndexCodec.java | 27 +------ .../salesforce/phoenix/util/IndexUtil.java | 27 +------ 3 files changed, 71 insertions(+), 58 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java b/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java index c5312209..6d3133a9 100644 --- a/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java +++ b/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java @@ -10,6 +10,10 @@ import java.util.Collections; import java.util.List; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.Writable; @@ -168,6 +172,7 @@ public static List deserialize(byte[] buf, int offset, int leng private final boolean isDataTableSalted; private final RowKeySchema dataRowKeySchema; + private List indexQualifiers; private int estimatedIndexRowKeyBytes; private int[][] dataRowKeyLocator; private int[] dataPkPosition; @@ -288,25 +293,58 @@ public byte[] buildRowKey(ValueGetter valueGetter, ImmutableBytesWritable rowKey } } - public List getCoverededColumns() { - return coveredColumns; + public List buildUpdateMutations(ValueGetter valueGetter, ImmutableBytesWritable dataRowKeyPtr) throws IOException { + return buildUpdateMutations(valueGetter, dataRowKeyPtr, HConstants.LATEST_TIMESTAMP); } - - public byte[] getEmptyKeyValueFamily() { - // Since the metadata of an index table will never change, - // we can infer this based on the family of the first covered column - // If if there are no covered columns, we know it's our default name - if (coveredColumns.isEmpty()) { - return QueryConstants.EMPTY_COLUMN_BYTES; + + @SuppressWarnings("deprecation") + public List buildUpdateMutations(ValueGetter valueGetter, ImmutableBytesWritable dataRowKeyPtr, long ts) throws IOException { + byte[] indexRowKey = this.buildRowKey(valueGetter, dataRowKeyPtr); + List mutations = Lists.newArrayListWithExpectedSize(2); + Delete delete = null; + Put put = new Put(indexRowKey); + mutations.add(put); + for (int i = 0; i < this.getCoverededColumns().size(); i++) { + ColumnReference ref = this.getCoverededColumns().get(i); + byte[] iq = this.indexQualifiers.get(i); + byte[] value = valueGetter.getLatestValue(ref); + if (value == null) { + if (delete == null) { + delete = new Delete(indexRowKey); + mutations.add(delete); + delete.setWriteToWAL(false); + } + delete.deleteColumns(ref.getFamily(), iq, ts); + } else { + put.add(ref.getFamily(), iq, ts, value); + } } - return coveredColumns.get(0).getFamily(); + // Add the empty key value + put.add(this.getEmptyKeyValueFamily(), QueryConstants.EMPTY_COLUMN_BYTES, ts, ByteUtil.EMPTY_BYTE_ARRAY); + put.setWriteToWAL(false); + return mutations; + } + + public Delete buildDeleteMutation(ValueGetter valueGetter, ImmutableBytesWritable dataRowKeyPtr) throws IOException { + return buildDeleteMutation(valueGetter, dataRowKeyPtr, HConstants.LATEST_TIMESTAMP); + } + + @SuppressWarnings("deprecation") + public Delete buildDeleteMutation(ValueGetter valueGetter, ImmutableBytesWritable dataRowKeyPtr, long ts) throws IOException { + byte[] indexRowKey = this.buildRowKey(valueGetter, dataRowKeyPtr); + Delete delete = new Delete(indexRowKey, ts, null); + delete.setWriteToWAL(false); + return delete; } public byte[] getIndexTableName() { return indexTableName; } - + public List getCoverededColumns() { + return coveredColumns; + } + public List getIndexedColumns() { return indexedColumns; } @@ -315,6 +353,16 @@ public List getAllColumns() { return allColumns; } + private byte[] getEmptyKeyValueFamily() { + // Since the metadata of an index table will never change, + // we can infer this based on the family of the first covered column + // If if there are no covered columns, we know it's our default name + if (coveredColumns.isEmpty()) { + return QueryConstants.EMPTY_COLUMN_BYTES; + } + return coveredColumns.get(0).getFamily(); + } + private RowKeyMetaData getRowKeyMetaData() { return rowKeyMetaData; } @@ -373,6 +421,11 @@ private int estimateIndexRowKeyByteSize() { * Init calculated state reading/creating */ private void initCachedState() { + indexQualifiers = Lists.newArrayListWithExpectedSize(this.coveredColumns.size()); + for (int i = 0; i < coveredColumns.size(); i++) { + ColumnReference ref = coveredColumns.get(i); + indexQualifiers.add(IndexUtil.getIndexColumnName(ref.getFamily(), ref.getQualifier())); + } estimatedIndexRowKeyBytes = estimateIndexRowKeyByteSize(); this.allColumns = Lists.newArrayListWithExpectedSize(indexedColumns.size() + coveredColumns.size()); diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java index 00077b45..889f163d 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java @@ -23,7 +23,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Mutation; -import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Pair; @@ -33,14 +32,11 @@ import com.salesforce.hbase.index.covered.IndexCodec; import com.salesforce.hbase.index.covered.IndexUpdate; import com.salesforce.hbase.index.covered.TableState; -import com.salesforce.hbase.index.covered.update.ColumnReference; import com.salesforce.hbase.index.scanner.Scanner; import com.salesforce.hbase.index.util.IndexManagementUtil; import com.salesforce.phoenix.cache.GlobalCache; import com.salesforce.phoenix.cache.IndexMetaDataCache; import com.salesforce.phoenix.cache.TenantCache; -import com.salesforce.phoenix.query.QueryConstants; -import com.salesforce.phoenix.util.ByteUtil; import com.salesforce.phoenix.util.ImmutableBytesPtr; import com.salesforce.phoenix.util.PhoenixRuntime; /** @@ -61,10 +57,6 @@ public void initialize(RegionCoprocessorEnvironment env) { this.conf = env.getConfiguration(); } - public List getIndexMaintainers() { - return indexMaintainers; - } - private List getIndexMaintainers(TableState state) { return getIndexMaintainers(state.getUpdateAttributes()); } @@ -100,7 +92,6 @@ private List getIndexMaintainers(Map attributes return indexMaintainers; } - @SuppressWarnings("deprecation") @Override public Iterable getIndexUpserts(TableState state) throws IOException { List indexMaintainers = getIndexMaintainers(state); @@ -117,25 +108,15 @@ public Iterable getIndexUpserts(TableState state) throws IOExceptio Scanner scanner = statePair.getFirst(); ValueGetter valueGetter = IndexManagementUtil.createGetterFromScanner(scanner, dataRowKey); ptr.set(dataRowKey); - byte[] rowKey = maintainer.buildRowKey(valueGetter, ptr); - Put put = new Put(rowKey); - put.setWriteToWAL(false); + // TODO: handle the case of a Put and Delete coming back + Mutation put = maintainer.buildUpdateMutations(valueGetter, ptr).get(0); indexUpdate.setTable(maintainer.getIndexTableName()); indexUpdate.setUpdate(put); - for (ColumnReference ref : maintainer.getCoverededColumns()) { - byte[] value = valueGetter.getLatestValue(ref); - if (value != null) { // FIXME: is this right? - put.add(ref.getFamily(), ref.getQualifier(), value); - } - } - // Add the empty key value - put.add(maintainer.getEmptyKeyValueFamily(), QueryConstants.EMPTY_COLUMN_BYTES, ByteUtil.EMPTY_BYTE_ARRAY); indexUpdates.add(indexUpdate); } return indexUpdates; } - @SuppressWarnings("deprecation") @Override public Iterable getIndexDeletes(TableState state) throws IOException { List indexMaintainers = getIndexMaintainers(state); @@ -153,9 +134,7 @@ public Iterable getIndexDeletes(TableState state) throws IOExceptio indexUpdate.setTable(maintainer.getIndexTableName()); ValueGetter valueGetter = IndexManagementUtil.createGetterFromScanner(scanner, dataRowKey); ptr.set(dataRowKey); - byte[] rowKey = maintainer.buildRowKey(valueGetter, ptr); - Delete delete = new Delete(rowKey); - delete.setWriteToWAL(false); + Delete delete = maintainer.buildDeleteMutation(valueGetter, ptr); indexUpdate.setUpdate(delete); indexUpdates.add(indexUpdate); } diff --git a/src/main/java/com/salesforce/phoenix/util/IndexUtil.java b/src/main/java/com/salesforce/phoenix/util/IndexUtil.java index 3eeb3e4e..2fb984fe 100644 --- a/src/main/java/com/salesforce/phoenix/util/IndexUtil.java +++ b/src/main/java/com/salesforce/phoenix/util/IndexUtil.java @@ -44,7 +44,6 @@ import com.salesforce.phoenix.exception.SQLExceptionCode; import com.salesforce.phoenix.exception.SQLExceptionInfo; import com.salesforce.phoenix.index.IndexMaintainer; -import com.salesforce.phoenix.query.QueryConstants; import com.salesforce.phoenix.schema.ColumnFamilyNotFoundException; import com.salesforce.phoenix.schema.ColumnNotFoundException; import com.salesforce.phoenix.schema.PColumn; @@ -125,7 +124,6 @@ public static PColumn getDataColumn(PTable dataTable, String indexColumnName) { } } - @SuppressWarnings("deprecation") public static List generateIndexData(PTable table, PTable index, List dataMutations, ImmutableBytesWritable ptr) throws SQLException { IndexMaintainer maintainer = index.getIndexMaintainer(table); List indexMutations = Lists.newArrayListWithExpectedSize(dataMutations.size()); @@ -157,30 +155,13 @@ public byte[] getLatestValue(ColumnReference ref) { }; // TODO: we could only handle a delete if maintainer.getIndexColumns().isEmpty(), // since the Delete marker will have no key values - assert(dataMutation instanceof Put); long ts = MetaDataUtil.getClientTimeStamp(dataMutation); ptr.set(dataMutation.getRow()); - byte[] indexRowKey = maintainer.buildRowKey(valueGetter, ptr); - Put put = new Put(indexRowKey); - for (ColumnReference ref : maintainer.getCoverededColumns()) { - try{ - byte[] value = valueGetter.getLatestValue(ref); - if (value != null) { - KeyValue kv = KeyValueUtil.newKeyValue(put.getRow(), ref.getFamily(), ref.getQualifier(), ts, value); - try { - put.add(kv); - } catch (IOException e) { - throw new SQLException(e); // Impossible - } - } - }catch(IOException e){ - throw new RuntimeException("Inmemory ValueGetter threw exception!",e); - } + try { + indexMutations.addAll(maintainer.buildUpdateMutations(valueGetter, ptr, ts)); + } catch (IOException e) { + throw new SQLException(e); } - put.add(maintainer.getEmptyKeyValueFamily(), QueryConstants.EMPTY_COLUMN_BYTES, ts, ByteUtil.EMPTY_BYTE_ARRAY); - put.setWriteToWAL(false); - - indexMutations.add(put); } } return indexMutations; From c615c8b391209e7d929367d27afa5e36828a2398 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Wed, 18 Sep 2013 08:16:19 -0700 Subject: [PATCH 041/102] Fix compiler errors (based on eclipse settings in phoenix/dev) --- .../hbase/index/covered/data/LocalTable.java | 20 ------------------- .../hbase/regionserver/ExposedMemStore.java | 3 +-- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/src/main/java/com/salesforce/hbase/index/covered/data/LocalTable.java b/src/main/java/com/salesforce/hbase/index/covered/data/LocalTable.java index 05584c64..c7a80fa3 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/data/LocalTable.java +++ b/src/main/java/com/salesforce/hbase/index/covered/data/LocalTable.java @@ -32,10 +32,8 @@ import java.util.List; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.regionserver.HRegion; @@ -52,7 +50,6 @@ */ public class LocalTable implements LocalHBaseState { - private volatile HTableInterface localTable; private RegionCoprocessorEnvironment env; public LocalTable(RegionCoprocessorEnvironment env) { @@ -76,21 +73,4 @@ public Result getCurrentRowState(Mutation m) throws IOException { scanner.close(); return r; } - - /** - * Ensure we have a connection to the local table. We need to do this after - * {@link #setup(RegionCoprocessorEnvironment)} because we are created on region startup and the - * table isn't actually accessible until later. - * @throws IOException if we can't reach the table - */ - private HTableInterface getLocalTable() throws IOException { - if (this.localTable == null) { - synchronized (this) { - if (this.localTable == null) { - localTable = env.getTable(env.getRegion().getTableDesc().getName()); - } - } - } - return this.localTable; - } } \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ExposedMemStore.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ExposedMemStore.java index 1f099e79..e2bdf7ee 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/ExposedMemStore.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ExposedMemStore.java @@ -8,8 +8,6 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KVComparator; -import com.google.common.primitives.Longs; - /** * A {@link MemStore} that exposes all the package-protected methods. *

@@ -21,6 +19,7 @@ public class ExposedMemStore extends MemStore { public static KVComparator IGNORE_MEMSTORE_TS_COMPARATOR = new KVComparator() { + @Override public int compare(final KeyValue left, final KeyValue right) { int ret = getRawComparator().compare(left.getBuffer(), left.getOffset() + KeyValue.ROW_OFFSET, From f066db78be39031a5f9e5dd1b93549da6048add0 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Wed, 18 Sep 2013 12:52:52 -0700 Subject: [PATCH 042/102] Fix caching of index maintainers on codec --- .../phoenix/cache/ServerCacheClient.java | 4 +- .../phoenix/execute/MutationState.java | 15 +-- .../phoenix/index/PhoenixIndexCodec.java | 43 ++++---- .../end2end/index/MutableIndexTest.java | 97 +++++++++++++++++-- 4 files changed, 125 insertions(+), 34 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/cache/ServerCacheClient.java b/src/main/java/com/salesforce/phoenix/cache/ServerCacheClient.java index 56b2b2d1..5ac39da0 100644 --- a/src/main/java/com/salesforce/phoenix/cache/ServerCacheClient.java +++ b/src/main/java/com/salesforce/phoenix/cache/ServerCacheClient.java @@ -151,7 +151,7 @@ public ServerCache addServerCache(ScanRanges keyRanges, final ImmutableBytesWrit closeables.add(chunk); ServerCache hashCacheSpec = null; SQLException firstException = null; - final byte[] cacheId = nextId(); + final byte[] cacheId = generateId(); /** * Execute EndPoint in parallel on each server to send compressed hash cache */ @@ -272,7 +272,7 @@ private void removeServerCache(byte[] cacheId, Set servers) throws S /** * Create an ID to keep the cached information across other operations independent */ - private static byte[] nextId() { + public static byte[] generateId() { return Bytes.toBytes(UUID.randomUUID().toString()); } } \ No newline at end of file diff --git a/src/main/java/com/salesforce/phoenix/execute/MutationState.java b/src/main/java/com/salesforce/phoenix/execute/MutationState.java index 94143e58..9f6be08a 100644 --- a/src/main/java/com/salesforce/phoenix/execute/MutationState.java +++ b/src/main/java/com/salesforce/phoenix/execute/MutationState.java @@ -46,6 +46,7 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.salesforce.phoenix.cache.ServerCacheClient; import com.salesforce.phoenix.cache.ServerCacheClient.ServerCache; import com.salesforce.phoenix.index.IndexMetaDataCacheClient; import com.salesforce.phoenix.index.PhoenixIndexCodec; @@ -337,21 +338,23 @@ public void commit() throws SQLException { ServerCache cache = null; if (hasIndexMaintainers && isDataTable) { - String attribName; - byte[] attribValue; + byte[] attribValue = null; + byte[] uuidValue; if (IndexMetaDataCacheClient.useIndexMetadataCache(mutations, tempPtr.getLength())) { IndexMetaDataCacheClient client = new IndexMetaDataCacheClient(connection, tableRef); cache = client.addIndexMetadataCache(mutations, tempPtr); - attribName = PhoenixIndexCodec.INDEX_UUID; - attribValue = cache.getId(); + uuidValue = cache.getId(); } else { - attribName = PhoenixIndexCodec.INDEX_MD; attribValue = ByteUtil.copyKeyBytesIfNecessary(tempPtr); + uuidValue = ServerCacheClient.generateId(); } // Either set the UUID to be able to access the index metadata from the cache // or set the index metadata directly on the Mutation for (Mutation mutation : mutations) { - mutation.setAttribute(attribName, attribValue); + mutation.setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue); + if (attribValue != null) { + mutation.setAttribute(PhoenixIndexCodec.INDEX_MD, attribValue); + } } } diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java index 889f163d..e3910627 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import com.google.common.collect.Lists; @@ -51,35 +52,38 @@ public class PhoenixIndexCodec implements IndexCodec { private List indexMaintainers; private final ImmutableBytesWritable ptr = new ImmutableBytesWritable(); private Configuration conf; + private byte[] uuid; @Override public void initialize(RegionCoprocessorEnvironment env) { this.conf = env.getConfiguration(); } - private List getIndexMaintainers(TableState state) { - return getIndexMaintainers(state.getUpdateAttributes()); + private List getIndexMaintainers() { + return indexMaintainers; } /** - * @param attributes attributes from a primary table update + * @param m mutation that is being processed * @return the {@link IndexMaintainer}s that would maintain the index for an update with the * attributes. */ - private List getIndexMaintainers(Map attributes) { - if (indexMaintainers != null) { - return indexMaintainers; - } - - byte[] md; + private boolean initIndexMaintainers(Mutation m) { + Map attributes = m.getAttributesMap(); byte[] uuid = attributes.get(INDEX_UUID); if (uuid == null) { - md = attributes.get(INDEX_MD); - if (md == null) { - indexMaintainers = Collections.emptyList(); - } else { - indexMaintainers = IndexMaintainer.deserialize(md); - } + this.uuid = null; + indexMaintainers = Collections.emptyList(); + return false; + } + if (this.uuid != null && Bytes.compareTo(this.uuid, uuid) == 0) { + return true; + } + this.uuid = uuid; + + byte[] md = attributes.get(INDEX_MD); + if (md != null) { + indexMaintainers = IndexMaintainer.deserialize(md); } else { byte[] tenantIdBytes = attributes.get(PhoenixRuntime.TENANT_ID_ATTRIB); ImmutableBytesWritable tenantId = @@ -89,12 +93,12 @@ private List getIndexMaintainers(Map attributes (IndexMetaDataCache) cache.getServerCache(new ImmutableBytesPtr(uuid)); this.indexMaintainers = indexCache.getIndexMaintainers(); } - return indexMaintainers; + return true; } @Override public Iterable getIndexUpserts(TableState state) throws IOException { - List indexMaintainers = getIndexMaintainers(state); + List indexMaintainers = getIndexMaintainers(); if (indexMaintainers.isEmpty()) { return Collections.emptyList(); } @@ -119,7 +123,7 @@ public Iterable getIndexUpserts(TableState state) throws IOExceptio @Override public Iterable getIndexDeletes(TableState state) throws IOException { - List indexMaintainers = getIndexMaintainers(state); + List indexMaintainers = getIndexMaintainers(); if (indexMaintainers.isEmpty()) { return Collections.emptyList(); } @@ -143,7 +147,6 @@ public Iterable getIndexDeletes(TableState state) throws IOExceptio @Override public boolean isEnabled(Mutation m) { - List maintainers = getIndexMaintainers(m.getAttributesMap()); - return maintainers.size() > 0; + return initIndexMaintainers(m); } } \ No newline at end of file diff --git a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java index 7385fe91..7d124b46 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java @@ -1,7 +1,5 @@ package com.salesforce.phoenix.end2end.index; -import static com.salesforce.phoenix.util.TestUtil.INDEX_DATA_SCHEMA; -import static com.salesforce.phoenix.util.TestUtil.INDEX_DATA_TABLE; import static com.salesforce.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -37,6 +35,8 @@ public class MutableIndexTest extends BaseHBaseManagedTimeTest{ private static final int INDEX_SPLITS = 4; private static final byte[] DATA_TABLE_FULL_NAME = SchemaUtil.getTableNameAsBytes(null, "T"); private static final byte[] INDEX_TABLE_FULL_NAME = SchemaUtil.getTableNameAsBytes(null, "I"); + public static final String MINDEX_DATA_SCHEMA = "MINDEX_TEST"; + public static final String MINDEX_DATA_TABLE = "MINDEX_DATA_TABLE"; @BeforeClass public static void doSetup() throws Exception { @@ -47,12 +47,36 @@ public static void doSetup() throws Exception { startServer(getUrl(), new ReadOnlyProps(props.entrySet().iterator())); } + private static void createTestTable() throws SQLException { + String ddl = " create table if not exists " + MINDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MINDEX_DATA_TABLE + "(" + + " varchar_pk VARCHAR NOT NULL, " + + " char_pk CHAR(5) NOT NULL, " + + " int_pk INTEGER NOT NULL, "+ + " long_pk BIGINT NOT NULL, " + + " decimal_pk DECIMAL(31, 10) NOT NULL, " + + " a.varchar_col1 VARCHAR, " + + " a.char_col1 CHAR(5), " + + " a.int_col1 INTEGER, " + + " a.long_col1 BIGINT, " + + " a.decimal_col1 DECIMAL(31, 10), " + + " b.varchar_col2 VARCHAR, " + + " b.char_col2 CHAR(5), " + + " b.int_col2 INTEGER, " + + " b.long_col2 BIGINT, " + + " b.decimal_col2 DECIMAL(31, 10) " + + " CONSTRAINT pk PRIMARY KEY (varchar_pk, char_pk, int_pk, long_pk DESC, decimal_pk))"; + Properties props = new Properties(TEST_PROPERTIES); + Connection conn = DriverManager.getConnection(getUrl(), props); + conn.createStatement().execute(ddl); + conn.close(); + } + // Populate the test table with data. private static void populateTestTable() throws SQLException { Properties props = new Properties(TEST_PROPERTIES); Connection conn = DriverManager.getConnection(getUrl(), props); try { - String upsert = "UPSERT INTO " + INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + INDEX_DATA_TABLE + String upsert = "UPSERT INTO " + MINDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MINDEX_DATA_TABLE + " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; PreparedStatement stmt = conn.prepareStatement(upsert); stmt.setString(1, "varchar1"); @@ -275,15 +299,15 @@ public void testIndexWithNullableFixedWithCols() throws Exception { Connection conn = DriverManager.getConnection(getUrl(), props); conn.setAutoCommit(false); try { - ensureTableCreated(getUrl(), INDEX_DATA_TABLE); + createTestTable(); populateTestTable(); - String ddl = "CREATE INDEX IDX ON " + INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + INDEX_DATA_TABLE + String ddl = "CREATE INDEX IDX ON " + MINDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MINDEX_DATA_TABLE + " (char_col1 ASC, int_col1 ASC)" + " INCLUDE (long_col1, long_col2)"; PreparedStatement stmt = conn.prepareStatement(ddl); stmt.execute(); - String query = "SELECT char_col1, int_col1 from " + INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + INDEX_DATA_TABLE; + String query = "SELECT char_col1, int_col1 from " + MINDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MINDEX_DATA_TABLE; ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + query); assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER INDEX_TEST.IDX", QueryUtil.getExplainPlan(rs)); @@ -302,4 +326,65 @@ public void testIndexWithNullableFixedWithCols() throws Exception { conn.close(); } } + + @Test + public void testCoveredColumnUpdates() throws Exception { + Properties props = new Properties(TEST_PROPERTIES); + Connection conn = DriverManager.getConnection(getUrl(), props); + conn.setAutoCommit(false); + try { + String tableName = MINDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MINDEX_DATA_TABLE; + createTestTable(); + populateTestTable(); + String ddl = "CREATE INDEX IDX ON " + tableName + + " (char_col1 ASC, int_col1 ASC)" + + " INCLUDE (long_col1, long_col2)"; + PreparedStatement stmt = conn.prepareStatement(ddl); + stmt.execute(); + + String query = "SELECT char_col1, int_col1, long_col2 from " +tableName; + ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + query); + assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER MINDEX_TEST.IDX", QueryUtil.getExplainPlan(rs)); + + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("chara", rs.getString(1)); + assertEquals(2, rs.getInt(2)); + assertEquals(3L, rs.getLong(3)); + assertTrue(rs.next()); + assertEquals("chara", rs.getString(1)); + assertEquals(3, rs.getInt(2)); + assertEquals(4L, rs.getLong(3)); + assertTrue(rs.next()); + assertEquals("chara", rs.getString(1)); + assertEquals(4, rs.getInt(2)); + assertEquals(5L, rs.getLong(3)); + assertFalse(rs.next()); + + stmt = conn.prepareStatement("UPSERT INTO " + tableName + + "(varchar_pk, char_pk, int_pk, long_pk , decimal_pk, long_col2) SELECT varchar_pk, char_pk, int_pk, long_pk , decimal_pk, long_col2+1 FROM " + + tableName + " WHERE long_col2=?"); + stmt.setLong(1,4L); + assertEquals(1,stmt.executeUpdate()); + conn.commit(); + + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("chara", rs.getString(1)); + assertEquals(2, rs.getInt(2)); + assertEquals(3L, rs.getLong(3)); + assertTrue(rs.next()); + assertEquals("chara", rs.getString(1)); + assertEquals(3, rs.getInt(2)); + assertEquals(5L, rs.getLong(3)); + assertTrue(rs.next()); + assertEquals("chara", rs.getString(1)); + assertEquals(4, rs.getInt(2)); + assertEquals(5L, rs.getLong(3)); + assertFalse(rs.next()); + + } finally { + conn.close(); + } + } } From bf839cf22ddbaebba75d0145ad0f8c05c7ee3b71 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Wed, 18 Sep 2013 13:31:14 -0700 Subject: [PATCH 043/102] Fix broken unit test --- .../com/salesforce/phoenix/end2end/index/MutableIndexTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java index 7d124b46..a56bbb5e 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java @@ -309,7 +309,7 @@ public void testIndexWithNullableFixedWithCols() throws Exception { String query = "SELECT char_col1, int_col1 from " + MINDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MINDEX_DATA_TABLE; ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER INDEX_TEST.IDX", QueryUtil.getExplainPlan(rs)); + assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER MINDEX_TEST.IDX", QueryUtil.getExplainPlan(rs)); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); From 692639d7d6ac11884029841bdfef0ee18c457076 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Wed, 18 Sep 2013 16:23:22 -0700 Subject: [PATCH 044/102] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3d4ba505..f31306e3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ![logo](http://forcedotcom.github.com/phoenix/images/logo.jpg) -Phoenix is a SQL skin over HBase, delivered as a client-embedded JDBC driver, powering the HBase use cases at Salesforce.com. Phoenix targets low-latency queries (milliseconds), as opposed to batch operation via map/reduce. To see what's supported, go to our [language reference guide](http://forcedotcom.github.com/phoenix/), read more on our [wiki](https://github.com/forcedotcom/phoenix/wiki), and download it [here](https://github.com/forcedotcom/phoenix/wiki/Download). +Phoenix is a SQL skin over HBase, delivered as a client-embedded JDBC driver, powering the HBase use cases at Salesforce.com. Phoenix targets low-latency queries (milliseconds), as opposed to batch operation via map/reduce. To see what's supported, go to our [language reference guide](http://forcedotcom.github.com/phoenix/), read more on our [wiki](https://github.com/forcedotcom/phoenix/wiki), take a look at our [FAQs](https://github.com/forcedotcom/phoenix/wiki/F.A.Q.), and download it [here](https://github.com/forcedotcom/phoenix/wiki/Download). ## Mission Become the standard means of accessing HBase data through a well-defined, industry standard API. @@ -42,6 +42,7 @@ Alternatively, you can build it yourself using maven by following these [build i ## Getting Started ## +Wanted to get started quickly? Take a look at our [FAQs](https://github.com/forcedotcom/phoenix/wiki/F.A.Q.) and take our quick start guide [here](https://github.com/forcedotcom/phoenix/wiki/Phoenix-in-15-minutes-or-less).

Command Line

From 25928c4a96c80deb5e7c8dbbc57c326e96fdc19f Mon Sep 17 00:00:00 2001 From: James Taylor Date: Wed, 18 Sep 2013 16:37:26 -0700 Subject: [PATCH 045/102] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f31306e3..69da9cc4 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ ![logo](http://forcedotcom.github.com/phoenix/images/logo.jpg) -Phoenix is a SQL skin over HBase, delivered as a client-embedded JDBC driver, powering the HBase use cases at Salesforce.com. Phoenix targets low-latency queries (milliseconds), as opposed to batch operation via map/reduce. To see what's supported, go to our [language reference guide](http://forcedotcom.github.com/phoenix/), read more on our [wiki](https://github.com/forcedotcom/phoenix/wiki), take a look at our [FAQs](https://github.com/forcedotcom/phoenix/wiki/F.A.Q.), and download it [here](https://github.com/forcedotcom/phoenix/wiki/Download). +Phoenix is a SQL skin over HBase, delivered as a client-embedded JDBC driver, powering the HBase use cases at Salesforce.com. Phoenix targets low-latency queries (milliseconds), as opposed to batch operation via map/reduce. To see what's supported, go to our [language reference guide](http://forcedotcom.github.com/phoenix/), read more on our [wiki](https://github.com/forcedotcom/phoenix/wiki), and download it [here](https://github.com/forcedotcom/phoenix/wiki/Download). ## Mission Become the standard means of accessing HBase data through a well-defined, industry standard API. ## Quick Start -Tired of reading already and just want to get started? Jump over to our quick start guide [here](https://github.com/forcedotcom/phoenix/wiki/Phoenix-in-15-minutes-or-less) or map to your existing HBase tables as described [here](https://github.com/forcedotcom/phoenix/wiki#wiki-mapping) and start querying now. +Tired of reading already and just want to get started? Listen to the Phoenix talks from [Hadoop Summit 2013](http://www.youtube.com/watch?v=YHsHdQ08trg) and [HBaseConn 2013](http://www.cloudera.com/content/cloudera/en/resources/library/hbasecon/hbasecon-2013--how-and-why-phoenix-puts-the-sql-back-into-nosql-video.html), check out our [FAQs](https://github.com/forcedotcom/phoenix/wiki/F.A.Q.), and jump over to our quick start guide [here](https://github.com/forcedotcom/phoenix/wiki/Phoenix-in-15-minutes-or-less) or map to your existing HBase tables as described [here](https://github.com/forcedotcom/phoenix/wiki#wiki-mapping) and start querying now. ## How It Works ## From 1de3f07eee4f45b6c613d2c9a67f068e321f8ab9 Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Wed, 18 Sep 2013 15:10:54 -0700 Subject: [PATCH 046/102] Lazy-load selected columns for current row state Before, we were loading all the columns of the row into memory. This could be wildly inefficient as we just care about a small subset of the columns (usually) for indexing. Now, we just load the columns the user requests per-index, on demand. This lead to an issue (esp when combined with the local HRegion reads) where two KVs with the same row-key (r,f,q,ts) would sort in the wrong order based on memstore timestamp. Resolved this with a custom KV comparator that always sorts the pending updates (as PendingKeyValue) before regular keyvalues, in the case that everything is the same. Really, we should probably also rip out the ExposedMemStore and go with a cleaner, simpler implementation. That's a V2 thing, but adding comments to this effect. --- .../salesforce/hbase/index/covered/Batch.java | 4 + .../hbase/index/covered/CoveredColumns.java | 59 +++++++ .../covered/CoveredColumnsIndexBuilder.java | 4 + .../hbase/index/covered/LocalTableState.java | 81 +++++++-- .../index/covered/data/LocalHBaseState.java | 8 +- .../hbase/index/covered/data/LocalTable.java | 10 +- .../hbase/regionserver/ExposedMemStore.java | 12 -- .../index/covered/TestLocalTableState.java | 159 ++++++++++++++++++ .../example/TestCoveredColumnIndexCodec.java | 5 +- 9 files changed, 317 insertions(+), 25 deletions(-) create mode 100644 src/main/java/com/salesforce/hbase/index/covered/CoveredColumns.java create mode 100644 src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java diff --git a/src/main/java/com/salesforce/hbase/index/covered/Batch.java b/src/main/java/com/salesforce/hbase/index/covered/Batch.java index 14647e43..1a3e9cca 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/Batch.java +++ b/src/main/java/com/salesforce/hbase/index/covered/Batch.java @@ -32,6 +32,8 @@ import org.apache.hadoop.hbase.KeyValue; +import com.salesforce.hbase.index.covered.LocalTableState.PendingKeyValue; + /** * A collection of {@link KeyValue KeyValues} to the primary table */ @@ -53,6 +55,8 @@ public void add(KeyValue kv){ if (pointDeleteCode != kv.getType()) { allPointDeletes = false; } + // wrap kvs with our special type so they get correctly added/rolled-back + kv = new PendingKeyValue(kv); batch.add(kv); } diff --git a/src/main/java/com/salesforce/hbase/index/covered/CoveredColumns.java b/src/main/java/com/salesforce/hbase/index/covered/CoveredColumns.java new file mode 100644 index 00000000..439c4aab --- /dev/null +++ b/src/main/java/com/salesforce/hbase/index/covered/CoveredColumns.java @@ -0,0 +1,59 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index.covered; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import com.salesforce.hbase.index.covered.update.ColumnReference; + +/** + * Manage a set of {@link ColumnReference}s for the {@link LocalTableState}. + */ +public class CoveredColumns { + + Set columns = new HashSet(); + + public Collection findNonCoveredColumns( + Collection columns2) { + List uncovered = new ArrayList(); + for (ColumnReference column : columns2) { + if (!columns.contains(column)) { + uncovered.add(column); + } + } + return uncovered; + } + + public void addColumn(ColumnReference column) { + this.columns.add(column); + } +} diff --git a/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java b/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java index 075dbab1..4f0407a4 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java +++ b/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java @@ -473,6 +473,10 @@ public Collection> getIndexUpdate(Delete d) throws IOExce batchMutationAndAddUpdates(updateMap, d); } + if (LOG.isDebugEnabled()) { + LOG.debug("Found index updates for Delete: " + updateMap); + } + return updateMap.toMap(); } diff --git a/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java b/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java index 5e7757a9..8b0aace3 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java +++ b/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java @@ -38,6 +38,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; @@ -45,6 +46,7 @@ import org.apache.hadoop.hbase.regionserver.KeyValueScanner; import org.apache.hadoop.hbase.util.Pair; +import com.google.common.primitives.Longs; import com.salesforce.hbase.index.covered.data.LocalHBaseState; import com.salesforce.hbase.index.covered.update.ColumnReference; import com.salesforce.hbase.index.covered.update.ColumnTracker; @@ -70,10 +72,64 @@ public class LocalTableState implements TableState { private LocalHBaseState table; private Mutation update; private Set trackedColumns = new HashSet(); - private boolean initialized; private ScannerBuilder scannerBuilder; private List kvs = new ArrayList(); private List hints; + private CoveredColumns columnSet; + + // TODO remove this when we replace the ExposedMemStore with our own implementation. Its really + // ugly to need to do this kind of thing, which is mostly due to the way the MemStore is written + // (e.g. we don't really care about read-points - everything we read from the HRegion is correctly + // visible). Leaving it for now as its a lot of work to replace that, though it could be a big + // win. + /** + * Just does a shallow copy of the wrapped keyvalue so we can mark it as a KeyValue to insert, + * rather than one already in the table + */ + static class PendingKeyValue extends KeyValue { + + /** + * @param kv to shallow copy + */ + public PendingKeyValue(KeyValue kv) { + super(kv.getBuffer(), kv.getOffset(), kv.getLength()); + this.setMemstoreTS(kv.getMemstoreTS()); + } + + } + + public static KVComparator PENDING_UPDATE_FIRST_COMPARATOR = new KVComparator() { + + @Override + public int compare(final KeyValue left, final KeyValue right) { + // do the raw comparison for kvs + int ret = + getRawComparator().compare(left.getBuffer(), left.getOffset() + KeyValue.ROW_OFFSET, + left.getKeyLength(), right.getBuffer(), right.getOffset() + KeyValue.ROW_OFFSET, + right.getKeyLength()); + if (ret != 0) return ret; + // then always ensure that our KVS sort first + if (left instanceof PendingKeyValue) { + if (right instanceof PendingKeyValue) { + // both pending, base it on the memstore ts + return compareMemstoreTs(left, right); + } + // left is, but right isn't so left sorts first + return -1; + } + // right is, but left isn't, so right sorts first + else if (right instanceof PendingKeyValue) { + return 1; + } + + // both are regular KVs, so we just do the usual MemStore compare + return compareMemstoreTs(left, right); + } + + private int compareMemstoreTs(KeyValue left, KeyValue right) { + return -Longs.compare(left.getMemstoreTS(), right.getMemstoreTS()); + } + }; public LocalTableState(RegionCoprocessorEnvironment environment, LocalHBaseState table, Mutation update) { this.env = environment; @@ -83,8 +139,9 @@ public LocalTableState(RegionCoprocessorEnvironment environment, LocalHBaseState // memstore Configuration conf = new Configuration(environment.getConfiguration()); ExposedMemStore.disableMemSLAB(conf); - this.memstore = new ExposedMemStore(conf, ExposedMemStore.IGNORE_MEMSTORE_TS_COMPARATOR); + this.memstore = new ExposedMemStore(conf, PENDING_UPDATE_FIRST_COMPARATOR); this.scannerBuilder = new ScannerBuilder(memstore, update); + this.columnSet = new CoveredColumns(); } public void addPendingUpdates(KeyValue... kvs) { @@ -131,7 +188,7 @@ public Set getTrackedColumns() { @Override public Pair getIndexedColumnsTableState( Collection indexedColumns) throws IOException { - ensureLocalStateInitialized(); + ensureLocalStateInitialized(indexedColumns); // filter out things with a newer timestamp and track the column references to which it applies ColumnTracker tracker = new ColumnTracker(indexedColumns); synchronized (this.trackedColumns) { @@ -156,7 +213,7 @@ public Pair getIndexedColumnsTableState( */ public Scanner getNonIndexedColumnsTableState(List columns) throws IOException { - ensureLocalStateInitialized(); + ensureLocalStateInitialized(columns); return this.scannerBuilder.buildNonIndexedColumnsScanner(columns, ts); } @@ -166,13 +223,17 @@ public Scanner getNonIndexedColumnsTableState(List co * Even then, there is still fairly low contention as each new Put/Delete will have its own table * state. */ - private synchronized void ensureLocalStateInitialized() throws IOException { - // check the local memstore - is it initialized? - if (!initialized){ - // add the current state of the row - this.addUpdate(this.table.getCurrentRowState(update).list()); - this.initialized = true; + private synchronized void ensureLocalStateInitialized( + Collection columns) throws IOException { + // check to see if we haven't initialized any columns yet + Collection toCover = this.columnSet.findNonCoveredColumns(columns); + // we have all the columns loaded, so we are good to go. + if (toCover.isEmpty()) { + return; } + + // add the current state of the row + this.addUpdate(this.table.getCurrentRowState(update, toCover).list()); } @Override diff --git a/src/main/java/com/salesforce/hbase/index/covered/data/LocalHBaseState.java b/src/main/java/com/salesforce/hbase/index/covered/data/LocalHBaseState.java index 5e487bb9..4e4f8062 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/data/LocalHBaseState.java +++ b/src/main/java/com/salesforce/hbase/index/covered/data/LocalHBaseState.java @@ -28,11 +28,14 @@ package com.salesforce.hbase.index.covered.data; import java.io.IOException; +import java.util.Collection; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Result; +import com.salesforce.hbase.index.covered.update.ColumnReference; + /** * Access the current state of the row in the local HBase table, given a mutation */ @@ -40,12 +43,15 @@ public interface LocalHBaseState { /** * @param m mutation for which we should get the current table state + * @param toCover all the columns the current row state needs to cover; hint the underlying lookup + * to save getting all the columns for the row * @return the full state of the given row. Includes all current versions (even if they are not * usually visible to the client (unless they are also doing a raw scan)). Never returns a * null {@link Result} - instead, when there is not data for the row, returns a * {@link Result} with no stored {@link KeyValue}s. * @throws IOException if there is an issue reading the row */ - public Result getCurrentRowState(Mutation m) throws IOException; + public Result getCurrentRowState(Mutation m, Collection toCover) + throws IOException; } \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/covered/data/LocalTable.java b/src/main/java/com/salesforce/hbase/index/covered/data/LocalTable.java index c7a80fa3..87871c40 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/data/LocalTable.java +++ b/src/main/java/com/salesforce/hbase/index/covered/data/LocalTable.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import org.apache.hadoop.hbase.KeyValue; @@ -39,6 +40,8 @@ import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.RegionScanner; +import com.salesforce.hbase.index.covered.update.ColumnReference; + /** * Wrapper around a lazily instantiated, local HTable. *

@@ -57,11 +60,16 @@ public LocalTable(RegionCoprocessorEnvironment env) { } @Override - public Result getCurrentRowState(Mutation m) throws IOException { + public Result getCurrentRowState(Mutation m, Collection columns) + throws IOException { byte[] row = m.getRow(); // need to use a scan here so we can get raw state, which Get doesn't provide. Scan s = new Scan(row, row); s.setRaw(true); + //add the necessary columns to the scan + for(ColumnReference column: columns){ + s.addFamily(column.getFamily()); + } s.setMaxVersions(); HRegion region = this.env.getRegion(); RegionScanner scanner = region.getScanner(s); diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ExposedMemStore.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ExposedMemStore.java index e2bdf7ee..b97be293 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/ExposedMemStore.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/ExposedMemStore.java @@ -17,18 +17,6 @@ */ public class ExposedMemStore extends MemStore { - public static KVComparator IGNORE_MEMSTORE_TS_COMPARATOR = new KVComparator() { - - @Override - public int compare(final KeyValue left, final KeyValue right) { - int ret = - getRawComparator().compare(left.getBuffer(), left.getOffset() + KeyValue.ROW_OFFSET, - left.getKeyLength(), right.getBuffer(), right.getOffset() + KeyValue.ROW_OFFSET, - right.getKeyLength()); - return ret; - } - }; - public ExposedMemStore(Configuration conf, KVComparator comparator) { super(conf, comparator); } diff --git a/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java b/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java new file mode 100644 index 00000000..fc688bcf --- /dev/null +++ b/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java @@ -0,0 +1,159 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index.covered; + +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.Type; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import com.salesforce.hbase.index.covered.LocalTableState.PendingKeyValue; +import com.salesforce.hbase.index.covered.data.LocalHBaseState; +import com.salesforce.hbase.index.covered.data.LocalTable; +import com.salesforce.hbase.index.covered.update.ColumnReference; +import com.salesforce.hbase.index.scanner.Scanner; + +/** + * + */ +public class TestLocalTableState { + + private static final byte[] row = Bytes.toBytes("row"); + private static final byte[] fam = Bytes.toBytes("fam"); + private static final byte[] qual = Bytes.toBytes("qual"); + private static final byte[] val = Bytes.toBytes("val"); + private static final long ts = 10; + + @Test + public void testCorrectOrderingWithLazyLoadingColumns() throws Exception { + Put m = new Put(row); + m.add(fam, qual, ts, val); + // setup mocks + Configuration conf = new Configuration(false); + RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class); + Mockito.when(env.getConfiguration()).thenReturn(conf); + + HRegion region = Mockito.mock(HRegion.class); + Mockito.when(env.getRegion()).thenReturn(region); + RegionScanner scanner = Mockito.mock(RegionScanner.class); + Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenReturn(scanner); + final byte[] stored = Bytes.toBytes("stored-value"); + Mockito.when(scanner.next(Mockito.any(List.class))).thenAnswer(new Answer() { + @Override + public Boolean answer(InvocationOnMock invocation) throws Throwable { + List list = (List) invocation.getArguments()[0]; + KeyValue kv = new KeyValue(row, fam, qual, ts, Type.Put, stored); + kv.setMemstoreTS(0); + list.add(kv); + return false; + } + }); + + + LocalHBaseState state = new LocalTable(env); + LocalTableState table = new LocalTableState(env, state, m); + //add the kvs from the mutation + table.addPendingUpdates(m.get(fam, qual)); + + // setup the lookup + ColumnReference col = new ColumnReference(fam, qual); + table.setCurrentTimestamp(ts); + //check that our value still shows up first on scan, even though this is a lazy load + Pair p = table.getIndexedColumnsTableState(Arrays.asList(col)); + Scanner s = p.getFirst(); + assertEquals("Didn't get the pending mutation's value first", m.get(fam, qual).get(0), s.next()); + } + + /** + * Test that we correctly rollback the state of a keyvalue if its a {@link PendingKeyValue}. + * @throws Exception + */ + @Test + @SuppressWarnings("unchecked") + public void testCorrectRollback() throws Exception { + Put m = new Put(row); + m.add(fam, qual, ts, val); + // setup mocks + Configuration conf = new Configuration(false); + RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class); + Mockito.when(env.getConfiguration()).thenReturn(conf); + Mockito.when(env.getConfiguration()).thenReturn(conf); + + HRegion region = Mockito.mock(HRegion.class); + Mockito.when(env.getRegion()).thenReturn(region); + RegionScanner scanner = Mockito.mock(RegionScanner.class); + Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenReturn(scanner); + final byte[] stored = Bytes.toBytes("stored-value"); + final KeyValue storedKv = new KeyValue(row, fam, qual, ts, Type.Put, stored); + storedKv.setMemstoreTS(2); + Mockito.when(scanner.next(Mockito.any(List.class))).thenAnswer(new Answer() { + @Override + public Boolean answer(InvocationOnMock invocation) throws Throwable { + List list = (List) invocation.getArguments()[0]; + + list.add(storedKv); + return false; + } + }); + LocalHBaseState state = new LocalTable(env); + LocalTableState table = new LocalTableState(env, state, m); + // add the kvs from the mutation + KeyValue kv = new LocalTableState.PendingKeyValue(m.get(fam, qual).get(0)); + kv.setMemstoreTS(0); + table.addPendingUpdates(kv); + + // setup the lookup + ColumnReference col = new ColumnReference(fam, qual); + table.setCurrentTimestamp(ts); + // check that the value is there + Pair p = table.getIndexedColumnsTableState(Arrays.asList(col)); + Scanner s = p.getFirst(); + assertEquals("Didn't get the pending mutation's value first", kv, s.next()); + + // rollback that value + table.rollback(Arrays.asList(kv)); + p = table.getIndexedColumnsTableState(Arrays.asList(col)); + s = p.getFirst(); + assertEquals("Didn't correctly rollback the row - still found it!", storedKv, s.next()); + } +} diff --git a/src/test/java/com/salesforce/hbase/index/covered/example/TestCoveredColumnIndexCodec.java b/src/test/java/com/salesforce/hbase/index/covered/example/TestCoveredColumnIndexCodec.java index c08a952b..10e583eb 100644 --- a/src/test/java/com/salesforce/hbase/index/covered/example/TestCoveredColumnIndexCodec.java +++ b/src/test/java/com/salesforce/hbase/index/covered/example/TestCoveredColumnIndexCodec.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; @@ -55,6 +56,7 @@ import com.salesforce.hbase.index.covered.LocalTableState; import com.salesforce.hbase.index.covered.data.LocalHBaseState; import com.salesforce.hbase.index.covered.example.CoveredColumnIndexCodec.ColumnEntry; +import com.salesforce.hbase.index.covered.update.ColumnReference; public class TestCoveredColumnIndexCodec { private static final byte[] PK = new byte[] { 'a' }; @@ -144,7 +146,8 @@ public SimpleTableState(Result r) { } @Override - public Result getCurrentRowState(Mutation m) throws IOException { + public Result getCurrentRowState(Mutation m, Collection toCover) + throws IOException { return r; } From da5ae7cba21909985d85eac324b4726fe22a6ad1 Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Wed, 18 Sep 2013 15:10:54 -0700 Subject: [PATCH 047/102] Add closing to scanners. Scanners were not being closed, causing a resource leak on the underlying MemStore. While this was fine for short operations (the ExposedMemStore was the only reference and it was let go after building the index updates for each mutation), it could be an issue for longer operations or operations that need to create a lot of scanners (e.g. lots of indexes). Now the scanners are allowed to be GC'ed. --- .../hbase/index/covered/example/CoveredColumnIndexCodec.java | 4 ++++ .../com/salesforce/hbase/index/scanner/EmptyScanner.java | 5 +++++ .../java/com/salesforce/hbase/index/scanner/Scanner.java | 3 ++- .../com/salesforce/hbase/index/scanner/ScannerBuilder.java | 5 +++++ .../java/com/salesforce/phoenix/index/PhoenixIndexCodec.java | 3 +++ 5 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexCodec.java b/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexCodec.java index 76991f77..aa51491f 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexCodec.java +++ b/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexCodec.java @@ -86,6 +86,8 @@ private IndexUpdate getIndexUpdateForGroup(ColumnGroup group, TableState state) Scanner kvs = stateInfo.getFirst(); Pair> columns = getNextEntries(refs, kvs, state.getCurrentRowKey()); + // make sure we close the scanner + kvs.close(); if (columns.getFirst().intValue() == 0) { return stateInfo.getSecond(); } @@ -143,6 +145,8 @@ private IndexUpdate getDeleteForGroup(ColumnGroup group, TableState state) { Pair kvs = state.getIndexedColumnsTableState(refs); Pair> columns = getNextEntries(refs, kvs.getFirst(), state.getCurrentRowKey()); + // make sure we close the scanner reference + kvs.getFirst().close(); // no change, just return the passed update if (columns.getFirst() == 0) { return kvs.getSecond(); diff --git a/src/main/java/com/salesforce/hbase/index/scanner/EmptyScanner.java b/src/main/java/com/salesforce/hbase/index/scanner/EmptyScanner.java index 3e1e9154..51cb8a36 100644 --- a/src/main/java/com/salesforce/hbase/index/scanner/EmptyScanner.java +++ b/src/main/java/com/salesforce/hbase/index/scanner/EmptyScanner.java @@ -24,4 +24,9 @@ public boolean seek(KeyValue next) throws IOException { public KeyValue peek() throws IOException { return null; } + + @Override + public void close() throws IOException { + // noop + } } \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/scanner/Scanner.java b/src/main/java/com/salesforce/hbase/index/scanner/Scanner.java index 1e696a4e..b412b995 100644 --- a/src/main/java/com/salesforce/hbase/index/scanner/Scanner.java +++ b/src/main/java/com/salesforce/hbase/index/scanner/Scanner.java @@ -1,5 +1,6 @@ package com.salesforce.hbase.index.scanner; +import java.io.Closeable; import java.io.IOException; import org.apache.hadoop.hbase.KeyValue; @@ -8,7 +9,7 @@ * Scan the primary table. This is similar to HBase's scanner, but ensures that you will never see * deleted columns/rows */ -public interface Scanner { +public interface Scanner extends Closeable { /** * @return the next keyvalue in the scanner or null if there is no next {@link KeyValue} diff --git a/src/main/java/com/salesforce/hbase/index/scanner/ScannerBuilder.java b/src/main/java/com/salesforce/hbase/index/scanner/ScannerBuilder.java index 8e49c709..4190db36 100644 --- a/src/main/java/com/salesforce/hbase/index/scanner/ScannerBuilder.java +++ b/src/main/java/com/salesforce/hbase/index/scanner/ScannerBuilder.java @@ -153,6 +153,11 @@ public boolean seek(KeyValue next) throws IOException { public KeyValue peek() throws IOException { return kvScanner.peek(); } + + @Override + public void close() { + kvScanner.close(); + } }; } } \ No newline at end of file diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java index e3910627..528c7b82 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java @@ -116,6 +116,8 @@ public Iterable getIndexUpserts(TableState state) throws IOExceptio Mutation put = maintainer.buildUpdateMutations(valueGetter, ptr).get(0); indexUpdate.setTable(maintainer.getIndexTableName()); indexUpdate.setUpdate(put); + //make sure we close the scanner when we are done + scanner.close(); indexUpdates.add(indexUpdate); } return indexUpdates; @@ -139,6 +141,7 @@ public Iterable getIndexDeletes(TableState state) throws IOExceptio ValueGetter valueGetter = IndexManagementUtil.createGetterFromScanner(scanner, dataRowKey); ptr.set(dataRowKey); Delete delete = maintainer.buildDeleteMutation(valueGetter, ptr); + scanner.close(); indexUpdate.setUpdate(delete); indexUpdates.add(indexUpdate); } From f6cbdc5c1ea10631af069cb726dbe9913a9dbb1d Mon Sep 17 00:00:00 2001 From: James Taylor Date: Thu, 19 Sep 2013 00:49:31 -0700 Subject: [PATCH 048/102] Add unit tests for setting covered column to new value and null, workaround the inability to pass through both a Put and Delete for a row, project only what's necessary in skip scan --- .../phoenix/cache/ServerCacheClient.java | 11 +- .../phoenix/index/IndexMaintainer.java | 37 ++- .../index/IndexMetaDataCacheClient.java | 4 +- .../phoenix/index/PhoenixIndexBuilder.java | 70 +++-- .../phoenix/index/PhoenixIndexCodec.java | 7 +- .../salesforce/phoenix/util/IndexUtil.java | 2 +- .../index/covered/TestLocalTableState.java | 1 + .../end2end/index/MutableIndexTest.java | 294 ++++++++++-------- 8 files changed, 237 insertions(+), 189 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/cache/ServerCacheClient.java b/src/main/java/com/salesforce/phoenix/cache/ServerCacheClient.java index 5ac39da0..173aac1c 100644 --- a/src/main/java/com/salesforce/phoenix/cache/ServerCacheClient.java +++ b/src/main/java/com/salesforce/phoenix/cache/ServerCacheClient.java @@ -36,8 +36,8 @@ import java.util.List; import java.util.Map; import java.util.NavigableMap; +import java.util.Random; import java.util.Set; -import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -75,7 +75,9 @@ * @since 0.1 */ public class ServerCacheClient { + public static final int UUID_LENGTH = Bytes.SIZEOF_LONG; private static final Log LOG = LogFactory.getLog(ServerCacheClient.class); + private static final Random RANDOM = new Random(); private final PhoenixConnection connection; private final TableRef cacheUsingTableRef; @@ -270,9 +272,12 @@ private void removeServerCache(byte[] cacheId, Set servers) throws S } /** - * Create an ID to keep the cached information across other operations independent + * Create an ID to keep the cached information across other operations independent. + * Using simple long random number, since the length of time we need this to be unique + * is very limited. */ public static byte[] generateId() { - return Bytes.toBytes(UUID.randomUUID().toString()); + long rand = RANDOM.nextLong(); + return Bytes.toBytes(rand); } } \ No newline at end of file diff --git a/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java b/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java index 6d3133a9..a665e370 100644 --- a/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java +++ b/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java @@ -12,10 +12,10 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; @@ -293,17 +293,39 @@ public byte[] buildRowKey(ValueGetter valueGetter, ImmutableBytesWritable rowKey } } - public List buildUpdateMutations(ValueGetter valueGetter, ImmutableBytesWritable dataRowKeyPtr) throws IOException { - return buildUpdateMutations(valueGetter, dataRowKeyPtr, HConstants.LATEST_TIMESTAMP); + // TODO: remove once Jesse handles a Put and Delete on the same row + @SuppressWarnings("deprecation") + public Put buildUpdateMutation(ValueGetter valueGetter, ImmutableBytesWritable dataRowKeyPtr) throws IOException { + byte[] indexRowKey = this.buildRowKey(valueGetter, dataRowKeyPtr); + Put put = new Put(indexRowKey); + for (int i = 0; i < this.getCoverededColumns().size(); i++) { + ColumnReference ref = this.getCoverededColumns().get(i); + byte[] iq = this.indexQualifiers.get(i); + byte[] value = valueGetter.getLatestValue(ref); + if (value == null) { + // TODO: we should use a Delete here, but Jesse's framework can't handle that yet. + // This will work, but will cause an otherwise sparse index to be bloated with empty + // values for any unset covered columns. + put.add(ref.getFamily(), iq, ByteUtil.EMPTY_BYTE_ARRAY); + } else { + put.add(ref.getFamily(), iq, value); + } + } + // Add the empty key value + put.add(this.getEmptyKeyValueFamily(), QueryConstants.EMPTY_COLUMN_BYTES, ByteUtil.EMPTY_BYTE_ARRAY); + put.setWriteToWAL(false); + return put; + } + + public Pair buildUpdateMutations(ValueGetter valueGetter, ImmutableBytesWritable dataRowKeyPtr) throws IOException { + return buildUpdateMutation(valueGetter, dataRowKeyPtr, HConstants.LATEST_TIMESTAMP); } @SuppressWarnings("deprecation") - public List buildUpdateMutations(ValueGetter valueGetter, ImmutableBytesWritable dataRowKeyPtr, long ts) throws IOException { + public Pair buildUpdateMutation(ValueGetter valueGetter, ImmutableBytesWritable dataRowKeyPtr, long ts) throws IOException { byte[] indexRowKey = this.buildRowKey(valueGetter, dataRowKeyPtr); - List mutations = Lists.newArrayListWithExpectedSize(2); Delete delete = null; Put put = new Put(indexRowKey); - mutations.add(put); for (int i = 0; i < this.getCoverededColumns().size(); i++) { ColumnReference ref = this.getCoverededColumns().get(i); byte[] iq = this.indexQualifiers.get(i); @@ -311,7 +333,6 @@ public List buildUpdateMutations(ValueGetter valueGetter, ImmutableByt if (value == null) { if (delete == null) { delete = new Delete(indexRowKey); - mutations.add(delete); delete.setWriteToWAL(false); } delete.deleteColumns(ref.getFamily(), iq, ts); @@ -322,7 +343,7 @@ public List buildUpdateMutations(ValueGetter valueGetter, ImmutableByt // Add the empty key value put.add(this.getEmptyKeyValueFamily(), QueryConstants.EMPTY_COLUMN_BYTES, ts, ByteUtil.EMPTY_BYTE_ARRAY); put.setWriteToWAL(false); - return mutations; + return new Pair(put,delete); } public Delete buildDeleteMutation(ValueGetter valueGetter, ImmutableBytesWritable dataRowKeyPtr) throws IOException { diff --git a/src/main/java/com/salesforce/phoenix/index/IndexMetaDataCacheClient.java b/src/main/java/com/salesforce/phoenix/index/IndexMetaDataCacheClient.java index e0bc9176..428288b9 100644 --- a/src/main/java/com/salesforce/phoenix/index/IndexMetaDataCacheClient.java +++ b/src/main/java/com/salesforce/phoenix/index/IndexMetaDataCacheClient.java @@ -15,8 +15,6 @@ public class IndexMetaDataCacheClient { private static final int USE_CACHE_THRESHOLD = 10; - // Would expect 128 bits as bytes would be 16 bytes, but these UUIDs end up being 36 bytes - private static final int UUID_BYTE_LENGTH = 36; private final ServerCacheClient serverCache; @@ -40,7 +38,7 @@ public IndexMetaDataCacheClient(PhoenixConnection connection, TableRef cacheUsin * @return */ public static boolean useIndexMetadataCache(List mutations, int indexMetaDataByteLength) { - return (indexMetaDataByteLength > UUID_BYTE_LENGTH && mutations.size() > USE_CACHE_THRESHOLD); + return (indexMetaDataByteLength > ServerCacheClient.UUID_LENGTH && mutations.size() > USE_CACHE_THRESHOLD); } /** diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java index 3aa4f302..0657fd68 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java @@ -1,29 +1,17 @@ /******************************************************************************* - * Copyright (c) 2013, Salesforce.com, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of Salesforce.com nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Copyright (c) 2013, Salesforce.com, Inc. All rights reserved. Redistribution and use in source and binary forms, with + * or without modification, are permitted provided that the following conditions are met: Redistributions of source code + * must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in + * binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. Neither the name of Salesforce.com nor the names + * of its contributors may be used to endorse or promote products derived from this software without specific prior + * written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************/ package com.salesforce.phoenix.index; @@ -42,6 +30,7 @@ import com.google.common.collect.Lists; import com.salesforce.hbase.index.covered.CoveredColumnsIndexBuilder; +import com.salesforce.hbase.index.covered.update.ColumnReference; import com.salesforce.phoenix.compile.ScanRanges; import com.salesforce.phoenix.query.KeyRange; import com.salesforce.phoenix.schema.PDataType; @@ -64,9 +53,18 @@ public void batchStarted(MiniBatchOperationInProgress> m } ScanRanges scanRanges = ScanRanges.create(Collections.singletonList(keys), SaltingUtil.VAR_BINARY_SCHEMA); Scan scan = new Scan(); - // TODO: when the index maintenance code starts projecting only what it needs in its scan - // we should do the same here. Since it reads the entire row, projecting here wouldn't be - // good, as then it wouldn't find everything it needs in the block cache. + // Project into scan only the columns we need to build the new index row and + // delete the old index row. We use the columns that we pass through for + // the Delete use case, as it includes indexed and covered columns as well + // as the empty key value column (which we use to detect a delete of the entire row). + List maintainers = getCodec().getIndexMaintainers(); + for (int i = 0; i < maintainers.size(); i++) { + IndexMaintainer maintainer = maintainers.get(i); + for (int j = 0; j < maintainer.getAllColumns().size(); j++) { + ColumnReference ref = maintainer.getAllColumns().get(j); + scan.addColumn(ref.getFamily(), ref.getQualifier()); + } + } scan.setFilter(scanRanges.getSkipScanFilter()); HRegion region = this.env.getRegion(); RegionScanner scanner = region.getScanner(scan); @@ -90,10 +88,14 @@ public void batchStarted(MiniBatchOperationInProgress> m } } } - - @Override - public boolean isEnabled(Mutation m) { - // ask the codec to see if we should even attempt indexing - return this.codec.isEnabled(m); - } + + private PhoenixIndexCodec getCodec() { + return (PhoenixIndexCodec)this.codec; + } + + @Override + public boolean isEnabled(Mutation m) { + // ask the codec to see if we should even attempt indexing + return this.codec.isEnabled(m); + } } \ No newline at end of file diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java index 528c7b82..e4a7f769 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; @@ -59,7 +60,7 @@ public void initialize(RegionCoprocessorEnvironment env) { this.conf = env.getConfiguration(); } - private List getIndexMaintainers() { + List getIndexMaintainers() { return indexMaintainers; } @@ -112,8 +113,8 @@ public Iterable getIndexUpserts(TableState state) throws IOExceptio Scanner scanner = statePair.getFirst(); ValueGetter valueGetter = IndexManagementUtil.createGetterFromScanner(scanner, dataRowKey); ptr.set(dataRowKey); - // TODO: handle the case of a Put and Delete coming back - Mutation put = maintainer.buildUpdateMutations(valueGetter, ptr).get(0); + // TODO: handle Pair because otherwise we'll bloat a sparse covered index + Put put = maintainer.buildUpdateMutation(valueGetter, ptr); indexUpdate.setTable(maintainer.getIndexTableName()); indexUpdate.setUpdate(put); //make sure we close the scanner when we are done diff --git a/src/main/java/com/salesforce/phoenix/util/IndexUtil.java b/src/main/java/com/salesforce/phoenix/util/IndexUtil.java index 2fb984fe..c1a71510 100644 --- a/src/main/java/com/salesforce/phoenix/util/IndexUtil.java +++ b/src/main/java/com/salesforce/phoenix/util/IndexUtil.java @@ -158,7 +158,7 @@ public byte[] getLatestValue(ColumnReference ref) { long ts = MetaDataUtil.getClientTimeStamp(dataMutation); ptr.set(dataMutation.getRow()); try { - indexMutations.addAll(maintainer.buildUpdateMutations(valueGetter, ptr, ts)); + indexMutations.add(maintainer.buildUpdateMutation(valueGetter, ptr, ts).getFirst()); } catch (IOException e) { throw new SQLException(e); } diff --git a/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java b/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java index fc688bcf..fc2af974 100644 --- a/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java +++ b/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java @@ -64,6 +64,7 @@ public class TestLocalTableState { private static final byte[] val = Bytes.toBytes("val"); private static final long ts = 10; + @SuppressWarnings("unchecked") @Test public void testCorrectOrderingWithLazyLoadingColumns() throws Exception { Put m = new Put(row); diff --git a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java index a56bbb5e..ea9c1759 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java @@ -16,6 +16,7 @@ import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -23,7 +24,6 @@ import com.salesforce.phoenix.end2end.BaseHBaseManagedTimeTest; import com.salesforce.phoenix.jdbc.PhoenixConnection; import com.salesforce.phoenix.query.ConnectionQueryServices; -import com.salesforce.phoenix.query.QueryConstants; import com.salesforce.phoenix.query.QueryServices; import com.salesforce.phoenix.util.QueryUtil; import com.salesforce.phoenix.util.ReadOnlyProps; @@ -33,10 +33,11 @@ public class MutableIndexTest extends BaseHBaseManagedTimeTest{ private static final int TABLE_SPLITS = 3; private static final int INDEX_SPLITS = 4; - private static final byte[] DATA_TABLE_FULL_NAME = SchemaUtil.getTableNameAsBytes(null, "T"); - private static final byte[] INDEX_TABLE_FULL_NAME = SchemaUtil.getTableNameAsBytes(null, "I"); - public static final String MINDEX_DATA_SCHEMA = "MINDEX_TEST"; - public static final String MINDEX_DATA_TABLE = "MINDEX_DATA_TABLE"; + public static final String SCHEMA_NAME = ""; + public static final String DATA_TABLE_NAME = "T"; + public static final String INDEX_TABLE_NAME = "I"; + private static final String DATA_TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME, "T"); + private static final String INDEX_TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME, "I"); @BeforeClass public static void doSetup() throws Exception { @@ -48,7 +49,7 @@ public static void doSetup() throws Exception { } private static void createTestTable() throws SQLException { - String ddl = " create table if not exists " + MINDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MINDEX_DATA_TABLE + "(" + + String ddl = "create table if not exists " + DATA_TABLE_FULL_NAME + "(" + " varchar_pk VARCHAR NOT NULL, " + " char_pk CHAR(5) NOT NULL, " + " int_pk INTEGER NOT NULL, "+ @@ -76,7 +77,7 @@ private static void populateTestTable() throws SQLException { Properties props = new Properties(TEST_PROPERTIES); Connection conn = DriverManager.getConnection(getUrl(), props); try { - String upsert = "UPSERT INTO " + MINDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MINDEX_DATA_TABLE + String upsert = "UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; PreparedStatement stmt = conn.prepareStatement(upsert); stmt.setString(1, "varchar1"); @@ -136,7 +137,8 @@ private static void populateTestTable() throws SQLException { } } - private static void destroyTables() throws Exception { + @Before + public void destroyTables() throws Exception { // Physically delete HBase table so that splits occur as expected for each test Properties props = new Properties(TEST_PROPERTIES); ConnectionQueryServices services = DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class).getQueryServices(); @@ -152,7 +154,7 @@ private static void destroyTables() throws Exception { admin.deleteTable(DATA_TABLE_FULL_NAME); } catch (TableNotFoundException e) { } - } finally { + } finally { admin.close(); } } @@ -173,124 +175,120 @@ public void testMutableTableIndexMaintanenceUnsalted() throws Exception { } private void testMutableTableIndexMaintanence(Integer tableSaltBuckets, Integer indexSaltBuckets) throws Exception { - try { - String query; - ResultSet rs; - - Properties props = new Properties(TEST_PROPERTIES); - Connection conn = DriverManager.getConnection(getUrl(), props); - conn.setAutoCommit(false); - conn.createStatement().execute("CREATE TABLE t (k VARCHAR NOT NULL PRIMARY KEY, v VARCHAR) " + (tableSaltBuckets == null ? "" : " SALT_BUCKETS=" + tableSaltBuckets)); - query = "SELECT * FROM t"; - rs = conn.createStatement().executeQuery(query); - assertFalse(rs.next()); - - conn.createStatement().execute("CREATE INDEX i ON t (v DESC)" + (indexSaltBuckets == null ? "" : " SALT_BUCKETS=" + indexSaltBuckets)); - query = "SELECT * FROM i"; - rs = conn.createStatement().executeQuery(query); - assertFalse(rs.next()); - - PreparedStatement stmt = conn.prepareStatement("UPSERT INTO t VALUES(?,?)"); - stmt.setString(1,"a"); - stmt.setString(2, "x"); - stmt.execute(); - stmt.setString(1,"b"); - stmt.setString(2, "y"); - stmt.execute(); - conn.commit(); - - query = "SELECT * FROM i"; - rs = conn.createStatement().executeQuery(query); - assertTrue(rs.next()); - assertEquals("y",rs.getString(1)); - assertEquals("b",rs.getString(2)); - assertTrue(rs.next()); - assertEquals("x",rs.getString(1)); - assertEquals("a",rs.getString(2)); - assertFalse(rs.next()); - - query = "SELECT k,v FROM t WHERE v = 'y'"; - rs = conn.createStatement().executeQuery(query); - assertTrue(rs.next()); - assertEquals("b",rs.getString(1)); - assertEquals("y",rs.getString(2)); - assertFalse(rs.next()); - - String expectedPlan; - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - expectedPlan = indexSaltBuckets == null ? - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER I 'y'" : - ("CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 KEYS OVER I 0...3,'y'\n" + - "CLIENT MERGE SORT"); - assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); - - // Will use index, so rows returned in DESC order. - // This is not a bug, though, because we can - // return in any order. - query = "SELECT k,v FROM t WHERE v >= 'x'"; - rs = conn.createStatement().executeQuery(query); - assertTrue(rs.next()); - assertEquals("b",rs.getString(1)); - assertEquals("y",rs.getString(2)); - assertTrue(rs.next()); - assertEquals("a",rs.getString(1)); - assertEquals("x",rs.getString(2)); - assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - expectedPlan = indexSaltBuckets == null ? - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER I (*-'x']" : - ("CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 RANGES OVER I 0...3,(*-'x']\n" + - "CLIENT MERGE SORT"); - assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); - - // Will still use index, since there's no LIMIT clause - query = "SELECT k,v FROM t WHERE v >= 'x' ORDER BY k"; - rs = conn.createStatement().executeQuery(query); - assertTrue(rs.next()); - assertEquals("a",rs.getString(1)); - assertEquals("x",rs.getString(2)); - assertTrue(rs.next()); - assertEquals("b",rs.getString(1)); - assertEquals("y",rs.getString(2)); - assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - // Turns into an ORDER BY, which could be bad if lots of data is - // being returned. Without stats we don't know. The alternative - // would be a full table scan. - expectedPlan = indexSaltBuckets == null ? - ("CLIENT PARALLEL 1-WAY RANGE SCAN OVER I (*-'x']\n" + - " SERVER TOP -1 ROWS SORTED BY [:K]\n" + - "CLIENT MERGE SORT") : - ("CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 RANGES OVER I 0...3,(*-'x']\n" + - " SERVER TOP -1 ROWS SORTED BY [:K]\n" + - "CLIENT MERGE SORT"); - assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); - - // Will use data table now, since there's a LIMIT clause and - // we're able to optimize out the ORDER BY, unless the data - // table is salted. - query = "SELECT k,v FROM t WHERE v >= 'x' ORDER BY k LIMIT 2"; - rs = conn.createStatement().executeQuery(query); - assertTrue(rs.next()); - assertEquals("a",rs.getString(1)); - assertEquals("x",rs.getString(2)); - assertTrue(rs.next()); - assertEquals("b",rs.getString(1)); - assertEquals("y",rs.getString(2)); - assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - expectedPlan = tableSaltBuckets == null ? - "CLIENT PARALLEL 1-WAY FULL SCAN OVER T\n" + - " SERVER FILTER BY V >= 'x'\n" + - " SERVER 2 ROW LIMIT\n" + - "CLIENT 2 ROW LIMIT" : - "CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 RANGES OVER I 0...3,(*-'x']\n" + - " SERVER TOP 2 ROWS SORTED BY [:K]\n" + - "CLIENT MERGE SORT"; - assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); - } finally { - destroyTables(); - } + String query; + ResultSet rs; + + Properties props = new Properties(TEST_PROPERTIES); + Connection conn = DriverManager.getConnection(getUrl(), props); + conn.setAutoCommit(false); + conn.createStatement().execute("CREATE TABLE " + DATA_TABLE_FULL_NAME + " (k VARCHAR NOT NULL PRIMARY KEY, v VARCHAR) " + (tableSaltBuckets == null ? "" : " SALT_BUCKETS=" + tableSaltBuckets)); + query = "SELECT * FROM " + DATA_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertFalse(rs.next()); + + conn.createStatement().execute("CREATE INDEX " + INDEX_TABLE_NAME + " ON " + DATA_TABLE_FULL_NAME + " (v DESC)" + (indexSaltBuckets == null ? "" : " SALT_BUCKETS=" + indexSaltBuckets)); + query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertFalse(rs.next()); + + PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?)"); + stmt.setString(1,"a"); + stmt.setString(2, "x"); + stmt.execute(); + stmt.setString(1,"b"); + stmt.setString(2, "y"); + stmt.execute(); + conn.commit(); + + query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("y",rs.getString(1)); + assertEquals("b",rs.getString(2)); + assertTrue(rs.next()); + assertEquals("x",rs.getString(1)); + assertEquals("a",rs.getString(2)); + assertFalse(rs.next()); + + query = "SELECT k,v FROM " + DATA_TABLE_FULL_NAME + " WHERE v = 'y'"; + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("b",rs.getString(1)); + assertEquals("y",rs.getString(2)); + assertFalse(rs.next()); + + String expectedPlan; + rs = conn.createStatement().executeQuery("EXPLAIN " + query); + expectedPlan = indexSaltBuckets == null ? + "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + INDEX_TABLE_FULL_NAME + " 'y'" : + ("CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 KEYS OVER " + INDEX_TABLE_FULL_NAME + " 0...3,'y'\n" + + "CLIENT MERGE SORT"); + assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); + + // Will use index, so rows returned in DESC order. + // This is not a bug, though, because we can + // return in any order. + query = "SELECT k,v FROM " + DATA_TABLE_FULL_NAME + " WHERE v >= 'x'"; + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("b",rs.getString(1)); + assertEquals("y",rs.getString(2)); + assertTrue(rs.next()); + assertEquals("a",rs.getString(1)); + assertEquals("x",rs.getString(2)); + assertFalse(rs.next()); + rs = conn.createStatement().executeQuery("EXPLAIN " + query); + expectedPlan = indexSaltBuckets == null ? + "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + INDEX_TABLE_FULL_NAME + " (*-'x']" : + ("CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 RANGES OVER " + INDEX_TABLE_FULL_NAME + " 0...3,(*-'x']\n" + + "CLIENT MERGE SORT"); + assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); + + // Will still use index, since there's no LIMIT clause + query = "SELECT k,v FROM " + DATA_TABLE_FULL_NAME + " WHERE v >= 'x' ORDER BY k"; + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("a",rs.getString(1)); + assertEquals("x",rs.getString(2)); + assertTrue(rs.next()); + assertEquals("b",rs.getString(1)); + assertEquals("y",rs.getString(2)); + assertFalse(rs.next()); + rs = conn.createStatement().executeQuery("EXPLAIN " + query); + // Turns into an ORDER BY, which could be bad if lots of data is + // being returned. Without stats we don't know. The alternative + // would be a full table scan. + expectedPlan = indexSaltBuckets == null ? + ("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + INDEX_TABLE_FULL_NAME + " (*-'x']\n" + + " SERVER TOP -1 ROWS SORTED BY [:K]\n" + + "CLIENT MERGE SORT") : + ("CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 RANGES OVER " + INDEX_TABLE_FULL_NAME + " 0...3,(*-'x']\n" + + " SERVER TOP -1 ROWS SORTED BY [:K]\n" + + "CLIENT MERGE SORT"); + assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); + + // Will use data table now, since there's a LIMIT clause and + // we're able to optimize out the ORDER BY, unless the data + // table is salted. + query = "SELECT k,v FROM " + DATA_TABLE_FULL_NAME + " WHERE v >= 'x' ORDER BY k LIMIT 2"; + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("a",rs.getString(1)); + assertEquals("x",rs.getString(2)); + assertTrue(rs.next()); + assertEquals("b",rs.getString(1)); + assertEquals("y",rs.getString(2)); + assertFalse(rs.next()); + rs = conn.createStatement().executeQuery("EXPLAIN " + query); + expectedPlan = tableSaltBuckets == null ? + "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + DATA_TABLE_FULL_NAME + "\n" + + " SERVER FILTER BY V >= 'x'\n" + + " SERVER 2 ROW LIMIT\n" + + "CLIENT 2 ROW LIMIT" : + "CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 RANGES OVER " + INDEX_TABLE_FULL_NAME + " 0...3,(*-'x']\n" + + " SERVER TOP 2 ROWS SORTED BY [:K]\n" + + "CLIENT MERGE SORT"; + assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); } @Test @@ -301,15 +299,15 @@ public void testIndexWithNullableFixedWithCols() throws Exception { try { createTestTable(); populateTestTable(); - String ddl = "CREATE INDEX IDX ON " + MINDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MINDEX_DATA_TABLE + String ddl = "CREATE INDEX " + INDEX_TABLE_NAME + " ON " + DATA_TABLE_FULL_NAME + " (char_col1 ASC, int_col1 ASC)" + " INCLUDE (long_col1, long_col2)"; PreparedStatement stmt = conn.prepareStatement(ddl); stmt.execute(); - String query = "SELECT char_col1, int_col1 from " + MINDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MINDEX_DATA_TABLE; + String query = "SELECT char_col1, int_col1 from " + DATA_TABLE_FULL_NAME; ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER MINDEX_TEST.IDX", QueryUtil.getExplainPlan(rs)); + assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + INDEX_TABLE_FULL_NAME, QueryUtil.getExplainPlan(rs)); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -333,18 +331,17 @@ public void testCoveredColumnUpdates() throws Exception { Connection conn = DriverManager.getConnection(getUrl(), props); conn.setAutoCommit(false); try { - String tableName = MINDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MINDEX_DATA_TABLE; createTestTable(); populateTestTable(); - String ddl = "CREATE INDEX IDX ON " + tableName + String ddl = "CREATE INDEX " + INDEX_TABLE_NAME + " ON " + DATA_TABLE_FULL_NAME + " (char_col1 ASC, int_col1 ASC)" + " INCLUDE (long_col1, long_col2)"; PreparedStatement stmt = conn.prepareStatement(ddl); stmt.execute(); - String query = "SELECT char_col1, int_col1, long_col2 from " +tableName; + String query = "SELECT char_col1, int_col1, long_col2 from " + DATA_TABLE_FULL_NAME; ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER MINDEX_TEST.IDX", QueryUtil.getExplainPlan(rs)); + assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + INDEX_TABLE_FULL_NAME, QueryUtil.getExplainPlan(rs)); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -361,9 +358,9 @@ public void testCoveredColumnUpdates() throws Exception { assertEquals(5L, rs.getLong(3)); assertFalse(rs.next()); - stmt = conn.prepareStatement("UPSERT INTO " + tableName - + "(varchar_pk, char_pk, int_pk, long_pk , decimal_pk, long_col2) SELECT varchar_pk, char_pk, int_pk, long_pk , decimal_pk, long_col2+1 FROM " - + tableName + " WHERE long_col2=?"); + stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + + "(varchar_pk, char_pk, int_pk, long_pk , decimal_pk, long_col2) SELECT varchar_pk, char_pk, int_pk, long_pk , decimal_pk, long_col2*2 FROM " + + DATA_TABLE_FULL_NAME + " WHERE long_col2=?"); stmt.setLong(1,4L); assertEquals(1,stmt.executeUpdate()); conn.commit(); @@ -376,7 +373,30 @@ public void testCoveredColumnUpdates() throws Exception { assertTrue(rs.next()); assertEquals("chara", rs.getString(1)); assertEquals(3, rs.getInt(2)); + assertEquals(8L, rs.getLong(3)); + assertTrue(rs.next()); + assertEquals("chara", rs.getString(1)); + assertEquals(4, rs.getInt(2)); assertEquals(5L, rs.getLong(3)); + assertFalse(rs.next()); + + stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + + "(varchar_pk, char_pk, int_pk, long_pk , decimal_pk, long_col2) SELECT varchar_pk, char_pk, int_pk, long_pk , decimal_pk, null FROM " + + DATA_TABLE_FULL_NAME + " WHERE long_col2=?"); + stmt.setLong(1,3L); + assertEquals(1,stmt.executeUpdate()); + conn.commit(); + + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("chara", rs.getString(1)); + assertEquals(2, rs.getInt(2)); + assertEquals(0, rs.getLong(3)); + assertTrue(rs.wasNull()); + assertTrue(rs.next()); + assertEquals("chara", rs.getString(1)); + assertEquals(3, rs.getInt(2)); + assertEquals(8L, rs.getLong(3)); assertTrue(rs.next()); assertEquals("chara", rs.getString(1)); assertEquals(4, rs.getInt(2)); From ea367b690338d86277aedda471e18baf475b5bf4 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Thu, 19 Sep 2013 01:32:42 -0700 Subject: [PATCH 049/102] Moved ImmutableBytesPtr into index.util package and use in place of ImmutableBytesWritable for Map keys --- .../index/builder/example/ColumnFamilyIndexer.java | 12 ++++++------ .../index/covered/update/IndexUpdateManager.java | 12 ++++++------ .../index}/util/ImmutableBytesPtr.java | 2 +- .../com/salesforce/phoenix/cache/GlobalCache.java | 2 +- .../java/com/salesforce/phoenix/cache/HashCache.java | 2 +- .../com/salesforce/phoenix/cache/TenantCache.java | 2 +- .../salesforce/phoenix/cache/TenantCacheImpl.java | 2 +- .../salesforce/phoenix/compile/DeleteCompiler.java | 2 +- .../salesforce/phoenix/compile/UpsertCompiler.java | 2 +- .../phoenix/coprocessor/HashJoinRegionScanner.java | 2 +- .../phoenix/coprocessor/MetaDataEndpointImpl.java | 2 +- .../phoenix/coprocessor/ScanProjector.java | 2 +- .../coprocessor/ServerCachingEndpointImpl.java | 2 +- .../salesforce/phoenix/execute/MutationState.java | 2 +- .../phoenix/expression/InListExpression.java | 2 +- .../aggregator/BaseDecimalStddevAggregator.java | 1 + .../expression/aggregator/BaseStddevAggregator.java | 2 +- .../DistinctValueWithCountClientAggregator.java | 2 +- .../DistinctValueWithCountServerAggregator.java | 1 + .../filter/MultiCQKeyValueComparisonFilter.java | 2 +- .../salesforce/phoenix/index/PhoenixIndexCodec.java | 2 +- .../salesforce/phoenix/join/HashCacheFactory.java | 1 + .../com/salesforce/phoenix/join/HashJoinInfo.java | 2 +- .../java/com/salesforce/phoenix/util/ByteUtil.java | 1 + .../java/com/salesforce/phoenix/util/TupleUtil.java | 1 + 25 files changed, 35 insertions(+), 30 deletions(-) rename src/main/java/com/salesforce/{phoenix => hbase/index}/util/ImmutableBytesPtr.java (98%) diff --git a/src/main/java/com/salesforce/hbase/index/builder/example/ColumnFamilyIndexer.java b/src/main/java/com/salesforce/hbase/index/builder/example/ColumnFamilyIndexer.java index 7d0afce7..3a1397eb 100644 --- a/src/main/java/com/salesforce/hbase/index/builder/example/ColumnFamilyIndexer.java +++ b/src/main/java/com/salesforce/hbase/index/builder/example/ColumnFamilyIndexer.java @@ -46,12 +46,12 @@ import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import com.salesforce.hbase.index.Indexer; import com.salesforce.hbase.index.builder.BaseIndexBuilder; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; /** * Simple indexer that just indexes rows based on their column families @@ -94,7 +94,7 @@ public static void enableIndexing(HTableDescriptor desc, Map fam Indexer.enableIndexing(desc, ColumnFamilyIndexer.class, opts); } - private Map columnTargetMap; + private Map columnTargetMap; @Override public void setup(RegionCoprocessorEnvironment env) { @@ -102,11 +102,11 @@ public void setup(RegionCoprocessorEnvironment env) { String[] families = conf.get(INDEX_TO_TABLE_COUNT_KEY).split(SEPARATOR); // build up our mapping of column - > index table - columnTargetMap = new HashMap(families.length); + columnTargetMap = new HashMap(families.length); for (int i = 0; i < families.length; i++) { byte[] fam = Bytes.toBytes(families[i]); String indexTable = conf.get(INDEX_TO_TABLE_CONF_PREFX + families[i]); - columnTargetMap.put(new ImmutableBytesWritable(fam), indexTable); + columnTargetMap.put(new ImmutableBytesPtr(fam), indexTable); } } @@ -121,7 +121,7 @@ public Collection> getIndexUpdate(Put p) { Set keys = p.getFamilyMap().keySet(); for (Entry> entry : p.getFamilyMap().entrySet()) { String ref = columnTargetMap - .get(new ImmutableBytesWritable(entry.getKey())); + .get(new ImmutableBytesPtr(entry.getKey())); // no reference for that column, skip it if (ref == null) { continue; @@ -172,7 +172,7 @@ public Collection> getIndexUpdate(Delete d) { Collection> updateMap = new ArrayList>(); for (Entry> entry : d.getFamilyMap().entrySet()) { String ref = columnTargetMap - .get(new ImmutableBytesWritable(entry.getKey())); + .get(new ImmutableBytesPtr(entry.getKey())); // no reference for that column, skip it if (ref == null) { continue; diff --git a/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java b/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java index fd667232..afbb53a1 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java +++ b/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java @@ -37,11 +37,11 @@ import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import com.google.common.collect.Lists; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; /** * Keeps track of the index updates @@ -104,8 +104,8 @@ private int comparePuts(Put p1, Put p2) { private static final String PHOENIX_HBASE_TEMP_DELETE_MARKER = "phoenix.hbase.temp.delete.marker"; private static final byte[] TRUE_MARKER = new byte[] { 1 }; - protected final Map> map = - new HashMap>(); + protected final Map> map = + new HashMap>(); /** * Add an index update. Keeps the latest {@link Put} for a given timestamp @@ -114,7 +114,7 @@ private int comparePuts(Put p1, Put p2) { */ public void addIndexUpdate(byte[] tableName, Mutation m) { // we only keep the most recent update - ImmutableBytesWritable key = new ImmutableBytesWritable(tableName); + ImmutableBytesPtr key = new ImmutableBytesPtr(tableName); Collection updates = map.get(key); if (updates == null) { updates = new SortedCollection(COMPARATOR); @@ -186,7 +186,7 @@ private void markMutationForRemoval(Mutation m) { public List> toMap() { List> updateMap = Lists.newArrayList(); - for (Entry> updates : map.entrySet()) { + for (Entry> updates : map.entrySet()) { // get is ok because we always set with just the bytes byte[] tableName = updates.getKey().get(); // TODO replace this as just storing a byte[], to avoid all the String <-> byte[] swapping @@ -219,7 +219,7 @@ private boolean shouldBeRemoved(Mutation m) { @Override public String toString() { StringBuffer sb = new StringBuffer("IndexUpdates:\n"); - for (Entry> entry : map.entrySet()) { + for (Entry> entry : map.entrySet()) { String tableName = Bytes.toString(entry.getKey().get()); sb.append(" " + tableName + ":\n"); for (Mutation m : entry.getValue()) { diff --git a/src/main/java/com/salesforce/phoenix/util/ImmutableBytesPtr.java b/src/main/java/com/salesforce/hbase/index/util/ImmutableBytesPtr.java similarity index 98% rename from src/main/java/com/salesforce/phoenix/util/ImmutableBytesPtr.java rename to src/main/java/com/salesforce/hbase/index/util/ImmutableBytesPtr.java index e8631ee3..52a3eadb 100644 --- a/src/main/java/com/salesforce/phoenix/util/ImmutableBytesPtr.java +++ b/src/main/java/com/salesforce/hbase/index/util/ImmutableBytesPtr.java @@ -25,7 +25,7 @@ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************/ -package com.salesforce.phoenix.util; +package com.salesforce.hbase.index.util; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; diff --git a/src/main/java/com/salesforce/phoenix/cache/GlobalCache.java b/src/main/java/com/salesforce/phoenix/cache/GlobalCache.java index b389b353..23d86b21 100644 --- a/src/main/java/com/salesforce/phoenix/cache/GlobalCache.java +++ b/src/main/java/com/salesforce/phoenix/cache/GlobalCache.java @@ -35,12 +35,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.memory.ChildMemoryManager; import com.salesforce.phoenix.memory.GlobalMemoryManager; import com.salesforce.phoenix.query.QueryServices; import com.salesforce.phoenix.query.QueryServicesOptions; import com.salesforce.phoenix.schema.PTable; -import com.salesforce.phoenix.util.ImmutableBytesPtr; /** diff --git a/src/main/java/com/salesforce/phoenix/cache/HashCache.java b/src/main/java/com/salesforce/phoenix/cache/HashCache.java index f05b6862..6b7ebe96 100644 --- a/src/main/java/com/salesforce/phoenix/cache/HashCache.java +++ b/src/main/java/com/salesforce/phoenix/cache/HashCache.java @@ -32,8 +32,8 @@ import org.apache.http.annotation.Immutable; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.schema.tuple.Tuple; -import com.salesforce.phoenix.util.ImmutableBytesPtr; /** diff --git a/src/main/java/com/salesforce/phoenix/cache/TenantCache.java b/src/main/java/com/salesforce/phoenix/cache/TenantCache.java index f825510d..75c49e04 100644 --- a/src/main/java/com/salesforce/phoenix/cache/TenantCache.java +++ b/src/main/java/com/salesforce/phoenix/cache/TenantCache.java @@ -32,9 +32,9 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.coprocessor.ServerCachingProtocol.ServerCacheFactory; import com.salesforce.phoenix.memory.MemoryManager; -import com.salesforce.phoenix.util.ImmutableBytesPtr; /** diff --git a/src/main/java/com/salesforce/phoenix/cache/TenantCacheImpl.java b/src/main/java/com/salesforce/phoenix/cache/TenantCacheImpl.java index fb4effe6..39f7f5c4 100644 --- a/src/main/java/com/salesforce/phoenix/cache/TenantCacheImpl.java +++ b/src/main/java/com/salesforce/phoenix/cache/TenantCacheImpl.java @@ -35,11 +35,11 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import com.google.common.cache.*; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.coprocessor.ServerCachingProtocol.ServerCacheFactory; import com.salesforce.phoenix.memory.MemoryManager; import com.salesforce.phoenix.memory.MemoryManager.MemoryChunk; import com.salesforce.phoenix.util.Closeables; -import com.salesforce.phoenix.util.ImmutableBytesPtr; /** * diff --git a/src/main/java/com/salesforce/phoenix/compile/DeleteCompiler.java b/src/main/java/com/salesforce/phoenix/compile/DeleteCompiler.java index 645eb624..bceb9a3d 100644 --- a/src/main/java/com/salesforce/phoenix/compile/DeleteCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/DeleteCompiler.java @@ -37,6 +37,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.compile.GroupByCompiler.GroupBy; import com.salesforce.phoenix.compile.OrderByCompiler.OrderBy; import com.salesforce.phoenix.coprocessor.UngroupedAggregateRegionObserver; @@ -52,7 +53,6 @@ import com.salesforce.phoenix.query.Scanner; import com.salesforce.phoenix.schema.*; import com.salesforce.phoenix.schema.tuple.Tuple; -import com.salesforce.phoenix.util.ImmutableBytesPtr; public class DeleteCompiler { private final PhoenixConnection connection; diff --git a/src/main/java/com/salesforce/phoenix/compile/UpsertCompiler.java b/src/main/java/com/salesforce/phoenix/compile/UpsertCompiler.java index 482c733d..8c22099b 100644 --- a/src/main/java/com/salesforce/phoenix/compile/UpsertCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/UpsertCompiler.java @@ -36,6 +36,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.compile.GroupByCompiler.GroupBy; import com.salesforce.phoenix.compile.OrderByCompiler.OrderBy; import com.salesforce.phoenix.coprocessor.UngroupedAggregateRegionObserver; @@ -54,7 +55,6 @@ import com.salesforce.phoenix.query.Scanner; import com.salesforce.phoenix.schema.*; import com.salesforce.phoenix.schema.tuple.Tuple; -import com.salesforce.phoenix.util.ImmutableBytesPtr; import com.salesforce.phoenix.util.SchemaUtil; public class UpsertCompiler { diff --git a/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java b/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java index 1d6de979..906f72a9 100644 --- a/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java +++ b/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java @@ -37,12 +37,12 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.regionserver.RegionScanner; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.cache.*; import com.salesforce.phoenix.join.HashJoinInfo; import com.salesforce.phoenix.parse.JoinTableNode.JoinType; import com.salesforce.phoenix.schema.tuple.ResultTuple; import com.salesforce.phoenix.schema.tuple.Tuple; -import com.salesforce.phoenix.util.ImmutableBytesPtr; import com.salesforce.phoenix.util.TupleUtil; public class HashJoinRegionScanner implements RegionScanner { diff --git a/src/main/java/com/salesforce/phoenix/coprocessor/MetaDataEndpointImpl.java b/src/main/java/com/salesforce/phoenix/coprocessor/MetaDataEndpointImpl.java index 101d2089..9708e144 100644 --- a/src/main/java/com/salesforce/phoenix/coprocessor/MetaDataEndpointImpl.java +++ b/src/main/java/com/salesforce/phoenix/coprocessor/MetaDataEndpointImpl.java @@ -76,6 +76,7 @@ import org.slf4j.LoggerFactory; import com.google.common.collect.Lists; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.cache.GlobalCache; import com.salesforce.phoenix.jdbc.PhoenixDatabaseMetaData; import com.salesforce.phoenix.query.QueryConstants; @@ -94,7 +95,6 @@ import com.salesforce.phoenix.schema.PTableType; import com.salesforce.phoenix.schema.TableNotFoundException; import com.salesforce.phoenix.util.ByteUtil; -import com.salesforce.phoenix.util.ImmutableBytesPtr; import com.salesforce.phoenix.util.KeyValueUtil; import com.salesforce.phoenix.util.MetaDataUtil; import com.salesforce.phoenix.util.SchemaUtil; diff --git a/src/main/java/com/salesforce/phoenix/coprocessor/ScanProjector.java b/src/main/java/com/salesforce/phoenix/coprocessor/ScanProjector.java index 4ada2fd7..062b7cc1 100644 --- a/src/main/java/com/salesforce/phoenix/coprocessor/ScanProjector.java +++ b/src/main/java/com/salesforce/phoenix/coprocessor/ScanProjector.java @@ -41,8 +41,8 @@ import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.io.WritableUtils; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.util.ByteUtil; -import com.salesforce.phoenix.util.ImmutableBytesPtr; import com.salesforce.phoenix.util.KeyValueUtil; public class ScanProjector { diff --git a/src/main/java/com/salesforce/phoenix/coprocessor/ServerCachingEndpointImpl.java b/src/main/java/com/salesforce/phoenix/coprocessor/ServerCachingEndpointImpl.java index 6afe3513..7c5abee9 100644 --- a/src/main/java/com/salesforce/phoenix/coprocessor/ServerCachingEndpointImpl.java +++ b/src/main/java/com/salesforce/phoenix/coprocessor/ServerCachingEndpointImpl.java @@ -32,9 +32,9 @@ import org.apache.hadoop.hbase.coprocessor.BaseEndpointCoprocessor; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.cache.GlobalCache; import com.salesforce.phoenix.cache.TenantCache; -import com.salesforce.phoenix.util.ImmutableBytesPtr; diff --git a/src/main/java/com/salesforce/phoenix/execute/MutationState.java b/src/main/java/com/salesforce/phoenix/execute/MutationState.java index 9f6be08a..b0f89647 100644 --- a/src/main/java/com/salesforce/phoenix/execute/MutationState.java +++ b/src/main/java/com/salesforce/phoenix/execute/MutationState.java @@ -46,6 +46,7 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.cache.ServerCacheClient; import com.salesforce.phoenix.cache.ServerCacheClient.ServerCache; import com.salesforce.phoenix.index.IndexMetaDataCacheClient; @@ -58,7 +59,6 @@ import com.salesforce.phoenix.schema.PTable; import com.salesforce.phoenix.schema.TableRef; import com.salesforce.phoenix.util.ByteUtil; -import com.salesforce.phoenix.util.ImmutableBytesPtr; import com.salesforce.phoenix.util.IndexUtil; import com.salesforce.phoenix.util.SQLCloseable; import com.salesforce.phoenix.util.ServerUtil; diff --git a/src/main/java/com/salesforce/phoenix/expression/InListExpression.java b/src/main/java/com/salesforce/phoenix/expression/InListExpression.java index 496cf55e..7cb4f88b 100644 --- a/src/main/java/com/salesforce/phoenix/expression/InListExpression.java +++ b/src/main/java/com/salesforce/phoenix/expression/InListExpression.java @@ -36,11 +36,11 @@ import org.apache.hadoop.io.WritableUtils; import com.google.common.collect.Sets; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.expression.visitor.ExpressionVisitor; import com.salesforce.phoenix.schema.PDataType; import com.salesforce.phoenix.schema.tuple.Tuple; import com.salesforce.phoenix.util.ByteUtil; -import com.salesforce.phoenix.util.ImmutableBytesPtr; /* * Implementation of a SQL foo IN (a,b,c) expression. Other than the first diff --git a/src/main/java/com/salesforce/phoenix/expression/aggregator/BaseDecimalStddevAggregator.java b/src/main/java/com/salesforce/phoenix/expression/aggregator/BaseDecimalStddevAggregator.java index e7774bfe..198d5a25 100644 --- a/src/main/java/com/salesforce/phoenix/expression/aggregator/BaseDecimalStddevAggregator.java +++ b/src/main/java/com/salesforce/phoenix/expression/aggregator/BaseDecimalStddevAggregator.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Pair; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.expression.ColumnExpression; import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.schema.ColumnModifier; diff --git a/src/main/java/com/salesforce/phoenix/expression/aggregator/BaseStddevAggregator.java b/src/main/java/com/salesforce/phoenix/expression/aggregator/BaseStddevAggregator.java index f36181b5..52e6840a 100644 --- a/src/main/java/com/salesforce/phoenix/expression/aggregator/BaseStddevAggregator.java +++ b/src/main/java/com/salesforce/phoenix/expression/aggregator/BaseStddevAggregator.java @@ -33,11 +33,11 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.schema.ColumnModifier; import com.salesforce.phoenix.schema.PDataType; import com.salesforce.phoenix.schema.tuple.Tuple; -import com.salesforce.phoenix.util.ImmutableBytesPtr; /** * diff --git a/src/main/java/com/salesforce/phoenix/expression/aggregator/DistinctValueWithCountClientAggregator.java b/src/main/java/com/salesforce/phoenix/expression/aggregator/DistinctValueWithCountClientAggregator.java index e3f67191..0fd67dd1 100644 --- a/src/main/java/com/salesforce/phoenix/expression/aggregator/DistinctValueWithCountClientAggregator.java +++ b/src/main/java/com/salesforce/phoenix/expression/aggregator/DistinctValueWithCountClientAggregator.java @@ -37,11 +37,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.exception.PhoenixIOException; import com.salesforce.phoenix.schema.ColumnModifier; import com.salesforce.phoenix.schema.PDataType; import com.salesforce.phoenix.schema.tuple.Tuple; -import com.salesforce.phoenix.util.ImmutableBytesPtr; /** * Client side Aggregator which will aggregate data and find distinct values with number of occurrences for each. diff --git a/src/main/java/com/salesforce/phoenix/expression/aggregator/DistinctValueWithCountServerAggregator.java b/src/main/java/com/salesforce/phoenix/expression/aggregator/DistinctValueWithCountServerAggregator.java index 1b672aa0..f02f0308 100644 --- a/src/main/java/com/salesforce/phoenix/expression/aggregator/DistinctValueWithCountServerAggregator.java +++ b/src/main/java/com/salesforce/phoenix/expression/aggregator/DistinctValueWithCountServerAggregator.java @@ -40,6 +40,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.query.QueryServices; import com.salesforce.phoenix.query.QueryServicesOptions; import com.salesforce.phoenix.schema.PDataType; diff --git a/src/main/java/com/salesforce/phoenix/filter/MultiCQKeyValueComparisonFilter.java b/src/main/java/com/salesforce/phoenix/filter/MultiCQKeyValueComparisonFilter.java index 55552970..215456cc 100644 --- a/src/main/java/com/salesforce/phoenix/filter/MultiCQKeyValueComparisonFilter.java +++ b/src/main/java/com/salesforce/phoenix/filter/MultiCQKeyValueComparisonFilter.java @@ -29,8 +29,8 @@ import org.apache.hadoop.hbase.util.Bytes; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.expression.Expression; -import com.salesforce.phoenix.util.ImmutableBytesPtr; /** * diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java index e4a7f769..9c92a4fe 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java @@ -35,11 +35,11 @@ import com.salesforce.hbase.index.covered.IndexUpdate; import com.salesforce.hbase.index.covered.TableState; import com.salesforce.hbase.index.scanner.Scanner; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.hbase.index.util.IndexManagementUtil; import com.salesforce.phoenix.cache.GlobalCache; import com.salesforce.phoenix.cache.IndexMetaDataCache; import com.salesforce.phoenix.cache.TenantCache; -import com.salesforce.phoenix.util.ImmutableBytesPtr; import com.salesforce.phoenix.util.PhoenixRuntime; /** * Phoenix-basec {@link IndexCodec}. Manages all the logic of how to cleanup an index ( diff --git a/src/main/java/com/salesforce/phoenix/join/HashCacheFactory.java b/src/main/java/com/salesforce/phoenix/join/HashCacheFactory.java index 420a9c07..7d69eb69 100644 --- a/src/main/java/com/salesforce/phoenix/join/HashCacheFactory.java +++ b/src/main/java/com/salesforce/phoenix/join/HashCacheFactory.java @@ -39,6 +39,7 @@ import org.apache.hadoop.io.WritableUtils; import org.xerial.snappy.Snappy; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.cache.HashCache; import com.salesforce.phoenix.coprocessor.ServerCachingProtocol.ServerCacheFactory; import com.salesforce.phoenix.expression.Expression; diff --git a/src/main/java/com/salesforce/phoenix/join/HashJoinInfo.java b/src/main/java/com/salesforce/phoenix/join/HashJoinInfo.java index 49e72d59..14dfb535 100644 --- a/src/main/java/com/salesforce/phoenix/join/HashJoinInfo.java +++ b/src/main/java/com/salesforce/phoenix/join/HashJoinInfo.java @@ -34,10 +34,10 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.io.WritableUtils; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.expression.ExpressionType; import com.salesforce.phoenix.parse.JoinTableNode.JoinType; -import com.salesforce.phoenix.util.ImmutableBytesPtr; public class HashJoinInfo { private static final String HASH_JOIN = "HashJoin"; diff --git a/src/main/java/com/salesforce/phoenix/util/ByteUtil.java b/src/main/java/com/salesforce/phoenix/util/ByteUtil.java index 8823f6fa..60b7cc2c 100644 --- a/src/main/java/com/salesforce/phoenix/util/ByteUtil.java +++ b/src/main/java/com/salesforce/phoenix/util/ByteUtil.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.WritableUtils; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.schema.ColumnModifier; diff --git a/src/main/java/com/salesforce/phoenix/util/TupleUtil.java b/src/main/java/com/salesforce/phoenix/util/TupleUtil.java index e7f82a03..ac65e44f 100644 --- a/src/main/java/com/salesforce/phoenix/util/TupleUtil.java +++ b/src/main/java/com/salesforce/phoenix/util/TupleUtil.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.WritableUtils; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.query.QueryConstants; import com.salesforce.phoenix.schema.tuple.Tuple; From 19b99d161c37bead34411ddbbd1c5406cab2eebc Mon Sep 17 00:00:00 2001 From: James Taylor Date: Thu, 19 Sep 2013 11:07:07 -0700 Subject: [PATCH 050/102] Change PERCENTILE_RANK to correct name of PERCENT_RANK --- docs/phoenix.csv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/phoenix.csv b/docs/phoenix.csv index 261f0751..14bf83b6 100644 --- a/docs/phoenix.csv +++ b/docs/phoenix.csv @@ -855,14 +855,14 @@ PERCENTILE_DISC is an inverse distribution function that assumes a discrete dist PERCENTILE_DISC( 0.9 ) WITHIN GROUP (ORDER BY X DESC) " -"Functions (Aggregate)","PERCENTILE_RANK"," -PERCENTILE_RANK( { numeric } ) WITHIN GROUP (ORDER BY { numericTerm } { ASC | DESC } ) +"Functions (Aggregate)","PERCENT_RANK"," +PERCENT_RANK( { numeric } ) WITHIN GROUP (ORDER BY { numericTerm } { ASC | DESC } ) "," The percentile rank for a hypothetical value, if inserted into the column. Aggregates are only allowed in select statements. The returned value is of decimal data type. "," -PERCENTILE_RANK( 100 ) WITHIN GROUP (ORDER BY X ASC) +PERCENT_RANK( 100 ) WITHIN GROUP (ORDER BY X ASC) " "Functions (Aggregate)","STDDEV_POP"," From 36b80085176eaa839c642e43c7849ca54fc27ba6 Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Thu, 19 Sep 2013 15:22:54 -0700 Subject: [PATCH 051/102] Adding custom memstore impl. We basically wrap a KeyValueSkipListSet, just like a regular MemStore, except we are (1) not dealing with (a) space considerations or (b) a snapshot set, and (2) ignoring memstore timestamps in favor of deciding when we want to overwrite keys based on how we obtain them. We can ignore the memstore timestamps because we know that anything we get from the local region is going to be MVCC visible - so it should just go in. However, we also want overwrite any existing state with our pending write that we are indexing, so that needs to clobber the KVs we get from the HRegion. This got really messy with a regular memstore as each KV from the MemStore frequently has a higher MemStoreTS, but we can't just up the pending KVs' MemStoreTs b/c a memstore relies on the MVCC readpoint, which generally is < Long.MAX_VALUE. By realizing that we don't need the snapshot or space requirements, we can go much faster than the previous implementation. Further, by being smart about how we manage the KVs, we can drop the extra object creation we were doing to wrap the pending KVs (which we did previously to ensure they sorted before the ones we got from the HRegion). --- .../salesforce/hbase/index/covered/Batch.java | 4 - .../hbase/index/covered/KeyValueStore.java | 43 +++ .../hbase/index/covered/LocalTableState.java | 71 +---- .../index/covered/data/IndexMemStore.java | 265 ++++++++++++++++++ .../scanner/FilteredKeyValueScanner.java | 6 +- .../hbase/index/scanner/ScannerBuilder.java | 7 +- .../IndexKeyValueSkipListSet.java | 86 ++++++ .../index/covered/TestLocalTableState.java | 3 +- .../index/covered/data/TestIndexMemStore.java | 70 +++++ 9 files changed, 483 insertions(+), 72 deletions(-) create mode 100644 src/main/java/com/salesforce/hbase/index/covered/KeyValueStore.java create mode 100644 src/main/java/com/salesforce/hbase/index/covered/data/IndexMemStore.java create mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/IndexKeyValueSkipListSet.java create mode 100644 src/test/java/com/salesforce/hbase/index/covered/data/TestIndexMemStore.java diff --git a/src/main/java/com/salesforce/hbase/index/covered/Batch.java b/src/main/java/com/salesforce/hbase/index/covered/Batch.java index 1a3e9cca..14647e43 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/Batch.java +++ b/src/main/java/com/salesforce/hbase/index/covered/Batch.java @@ -32,8 +32,6 @@ import org.apache.hadoop.hbase.KeyValue; -import com.salesforce.hbase.index.covered.LocalTableState.PendingKeyValue; - /** * A collection of {@link KeyValue KeyValues} to the primary table */ @@ -55,8 +53,6 @@ public void add(KeyValue kv){ if (pointDeleteCode != kv.getType()) { allPointDeletes = false; } - // wrap kvs with our special type so they get correctly added/rolled-back - kv = new PendingKeyValue(kv); batch.add(kv); } diff --git a/src/main/java/com/salesforce/hbase/index/covered/KeyValueStore.java b/src/main/java/com/salesforce/hbase/index/covered/KeyValueStore.java new file mode 100644 index 00000000..88a99cce --- /dev/null +++ b/src/main/java/com/salesforce/hbase/index/covered/KeyValueStore.java @@ -0,0 +1,43 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index.covered; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.regionserver.KeyValueScanner; + +/** + * Store a collection of KeyValues in memory. + */ +public interface KeyValueStore { + + public void add(KeyValue kv, boolean overwrite); + + public KeyValueScanner getScanner(); + + public void rollback(KeyValue kv); +} \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java b/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java index 8b0aace3..d7598664 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java +++ b/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java @@ -38,7 +38,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; @@ -46,7 +45,7 @@ import org.apache.hadoop.hbase.regionserver.KeyValueScanner; import org.apache.hadoop.hbase.util.Pair; -import com.google.common.primitives.Longs; +import com.salesforce.hbase.index.covered.data.IndexMemStore; import com.salesforce.hbase.index.covered.data.LocalHBaseState; import com.salesforce.hbase.index.covered.update.ColumnReference; import com.salesforce.hbase.index.covered.update.ColumnTracker; @@ -68,7 +67,7 @@ public class LocalTableState implements TableState { private long ts; private RegionCoprocessorEnvironment env; - private ExposedMemStore memstore; + private KeyValueStore memstore; private LocalHBaseState table; private Mutation update; private Set trackedColumns = new HashSet(); @@ -77,60 +76,6 @@ public class LocalTableState implements TableState { private List hints; private CoveredColumns columnSet; - // TODO remove this when we replace the ExposedMemStore with our own implementation. Its really - // ugly to need to do this kind of thing, which is mostly due to the way the MemStore is written - // (e.g. we don't really care about read-points - everything we read from the HRegion is correctly - // visible). Leaving it for now as its a lot of work to replace that, though it could be a big - // win. - /** - * Just does a shallow copy of the wrapped keyvalue so we can mark it as a KeyValue to insert, - * rather than one already in the table - */ - static class PendingKeyValue extends KeyValue { - - /** - * @param kv to shallow copy - */ - public PendingKeyValue(KeyValue kv) { - super(kv.getBuffer(), kv.getOffset(), kv.getLength()); - this.setMemstoreTS(kv.getMemstoreTS()); - } - - } - - public static KVComparator PENDING_UPDATE_FIRST_COMPARATOR = new KVComparator() { - - @Override - public int compare(final KeyValue left, final KeyValue right) { - // do the raw comparison for kvs - int ret = - getRawComparator().compare(left.getBuffer(), left.getOffset() + KeyValue.ROW_OFFSET, - left.getKeyLength(), right.getBuffer(), right.getOffset() + KeyValue.ROW_OFFSET, - right.getKeyLength()); - if (ret != 0) return ret; - // then always ensure that our KVS sort first - if (left instanceof PendingKeyValue) { - if (right instanceof PendingKeyValue) { - // both pending, base it on the memstore ts - return compareMemstoreTs(left, right); - } - // left is, but right isn't so left sorts first - return -1; - } - // right is, but left isn't, so right sorts first - else if (right instanceof PendingKeyValue) { - return 1; - } - - // both are regular KVs, so we just do the usual MemStore compare - return compareMemstoreTs(left, right); - } - - private int compareMemstoreTs(KeyValue left, KeyValue right) { - return -Longs.compare(left.getMemstoreTS(), right.getMemstoreTS()); - } - }; - public LocalTableState(RegionCoprocessorEnvironment environment, LocalHBaseState table, Mutation update) { this.env = environment; this.table = table; @@ -139,7 +84,7 @@ public LocalTableState(RegionCoprocessorEnvironment environment, LocalHBaseState // memstore Configuration conf = new Configuration(environment.getConfiguration()); ExposedMemStore.disableMemSLAB(conf); - this.memstore = new ExposedMemStore(conf, PENDING_UPDATE_FIRST_COMPARATOR); + this.memstore = new IndexMemStore(IndexMemStore.COMPARATOR); this.scannerBuilder = new ScannerBuilder(memstore, update); this.columnSet = new CoveredColumns(); } @@ -156,9 +101,13 @@ public void addPendingUpdates(List kvs) { } private void addUpdate(List list) { + addUpdate(list, true); + } + + private void addUpdate(List list, boolean overwrite) { if (list == null) return; for (KeyValue kv : list) { - this.memstore.add(kv); + this.memstore.add(kv, overwrite); } } @@ -233,7 +182,7 @@ private synchronized void ensureLocalStateInitialized( } // add the current state of the row - this.addUpdate(this.table.getCurrentRowState(update, toCover).list()); + this.addUpdate(this.table.getCurrentRowState(update, toCover).list(), false); } @Override @@ -247,7 +196,7 @@ public byte[] getCurrentRowKey() { } public Result getCurrentRowState() { - KeyValueScanner scanner = this.memstore.getScanners().get(0); + KeyValueScanner scanner = this.memstore.getScanner(); List kvs = new ArrayList(); while (scanner.peek() != null) { try { diff --git a/src/main/java/com/salesforce/hbase/index/covered/data/IndexMemStore.java b/src/main/java/com/salesforce/hbase/index/covered/data/IndexMemStore.java new file mode 100644 index 00000000..73b5c007 --- /dev/null +++ b/src/main/java/com/salesforce/hbase/index/covered/data/IndexMemStore.java @@ -0,0 +1,265 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index.covered.data; + +import java.util.Comparator; +import java.util.Iterator; +import java.util.SortedSet; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.KeyComparator; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.IndexKeyValueSkipListSet; +import org.apache.hadoop.hbase.regionserver.KeyValueScanner; +import org.apache.hadoop.hbase.regionserver.MemStore; +import org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner; +import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; + +import com.salesforce.hbase.index.covered.KeyValueStore; + +/** + * Like the HBase {@link MemStore}, but without all that exta work around maintaining snapshots and + * sizing (for right now). We still support the concurrent access (in case indexes are built in + * parallel). Essentially, this is a light wrapper around a ConcurrentSkipListMap. + */ +public class IndexMemStore implements KeyValueStore { + + private IndexKeyValueSkipListSet kvset; + private TimeRangeTracker timeRangeTracker = new TimeRangeTracker(); + private Comparator comparator; + + /** + * Compare two {@link KeyValue}s based only on their row keys. Similar to the standard + * {@link KeyValue#COMPARATOR}, but doesn't take into consideration the memstore timestamps. We + * instead manage which KeyValue to retain based on how its loaded here + */ + public static Comparator COMPARATOR = new Comparator() { + + private final KeyComparator rawcomparator = new KeyComparator(); + + @Override + public int compare(final KeyValue left, final KeyValue right) { + return rawcomparator.compare(left.getBuffer(), left.getOffset() + KeyValue.ROW_OFFSET, + left.getKeyLength(), right.getBuffer(), right.getOffset() + KeyValue.ROW_OFFSET, + right.getKeyLength()); + } + }; + + /** + * Create a store with the given comparator. This comparator is used to determine both sort order + * as well as equality of {@link KeyValue}s. + * @param comparator to use + */ + public IndexMemStore(Comparator comparator) { + this.comparator = comparator; + this.kvset = IndexKeyValueSkipListSet.create(comparator); + } + + @Override + public void add(KeyValue kv, boolean overwrite) { + // if overwriting, we will always update + boolean updated = true; + if (!overwrite) { + // null if there was no previous value, so we added the kv + updated = (kvset.putIfAbsent(kv) == null); + } else { + kvset.add(kv); + } + + // if we updated, we need to do some tracking work + if (updated) { + // update the max timestamp + this.timeRangeTracker.includeTimestamp(kv); + } + } + + @Override + public void rollback(KeyValue kv) { + // If the key is in the store, delete it + this.kvset.remove(kv); + } + + @Override + public KeyValueScanner getScanner() { + return new MemStoreScanner(); + } + + /* + * MemStoreScanner implements the KeyValueScanner. It lets the caller scan the contents of a + * memstore -- both current map and snapshot. This behaves as if it were a real scanner but does + * not maintain position. + */ + // This class is adapted from org.apache.hadoop.hbase.MemStore.MemStoreScanner, HBase 0.94.12 + // It does basically the same thing as the MemStoreScanner, but it only keeps track of a single + // set, rather than a primary and a secondary set of KeyValues. + protected class MemStoreScanner extends NonLazyKeyValueScanner { + // Next row information for the set + private KeyValue nextRow = null; + + // last iterated KVs for kvset and snapshot (to restore iterator state after reseek) + private KeyValue kvsetItRow = null; + + // iterator based scanning. + private Iterator kvsetIt; + + // The kvset at the time of creating this scanner + volatile IndexKeyValueSkipListSet kvsetAtCreation; + + MemStoreScanner() { + super(); + kvsetAtCreation = kvset; + } + + private KeyValue getNext(Iterator it) { + // in the original implementation we cared about the current thread's readpoint from MVCC. + // However, we don't need to worry here because everything the index can see, is also visible + // to the client (or is the pending primary table update, so it will be once the index is + // written, so it might as well be). + KeyValue v = null; + try { + while (it.hasNext()) { + v = it.next(); + return v; + } + + return null; + } finally { + if (v != null) { + kvsetItRow = v; + } + } + } + + /** + * Set the scanner at the seek key. Must be called only once: there is no thread safety between + * the scanner and the memStore. + * @param key seek value + * @return false if the key is null or if there is no data + */ + @Override + public synchronized boolean seek(KeyValue key) { + if (key == null) { + close(); + return false; + } + + // kvset and snapshot will never be null. + // if tailSet can't find anything, SortedSet is empty (not null). + kvsetIt = kvsetAtCreation.tailSet(key).iterator(); + kvsetItRow = null; + + return seekInSubLists(key); + } + + /** + * (Re)initialize the iterators after a seek or a reseek. + */ + private synchronized boolean seekInSubLists(KeyValue key) { + nextRow = getNext(kvsetIt); + return nextRow != null; + } + + /** + * Move forward on the sub-lists set previously by seek. + * @param key seek value (should be non-null) + * @return true if there is at least one KV to read, false otherwise + */ + @Override + public synchronized boolean reseek(KeyValue key) { + /* + * See HBASE-4195 & HBASE-3855 & HBASE-6591 for the background on this implementation. This + * code is executed concurrently with flush and puts, without locks. Two points must be known + * when working on this code: 1) It's not possible to use the 'kvTail' and 'snapshot' + * variables, as they are modified during a flush. 2) The ideal implementation for performance + * would use the sub skip list implicitly pointed by the iterators 'kvsetIt' and 'snapshotIt'. + * Unfortunately the Java API does not offer a method to get it. So we remember the last keys + * we iterated to and restore the reseeked set to at least that point. + */ + + kvsetIt = kvsetAtCreation.tailSet(getHighest(key, kvsetItRow)).iterator(); + return seekInSubLists(key); + } + + /* + * Returns the higher of the two key values, or null if they are both null. This uses + * comparator.compare() to compare the KeyValue using the memstore comparator. + */ + private KeyValue getHighest(KeyValue first, KeyValue second) { + if (first == null && second == null) { + return null; + } + if (first != null && second != null) { + int compare = comparator.compare(first, second); + return (compare > 0 ? first : second); + } + return (first != null ? first : second); + } + + @Override + public synchronized KeyValue peek() { + // DebugPrint.println(" MS@" + hashCode() + " peek = " + getLowest()); + return nextRow; + } + + @Override + public synchronized KeyValue next() { + if (nextRow == null) { + return null; + } + + final KeyValue ret = nextRow; + + // Advance the iterators + nextRow = getNext(kvsetIt); + + return ret; + } + + @Override + public synchronized void close() { + this.nextRow = null; + this.kvsetIt = null; + this.kvsetItRow = null; + } + + /** + * MemStoreScanner returns max value as sequence id because it will always have the latest data + * among all files. + */ + @Override + public long getSequenceID() { + return Long.MAX_VALUE; + } + + @Override + public boolean shouldUseScanner(Scan scan, SortedSet columns, long oldestUnexpiredTS) { + return (timeRangeTracker.includesTimeRange(scan.getTimeRange()) && (timeRangeTracker + .getMaximumTimestamp() >= oldestUnexpiredTS)); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/scanner/FilteredKeyValueScanner.java b/src/main/java/com/salesforce/hbase/index/scanner/FilteredKeyValueScanner.java index 377ac8b4..af6d993b 100644 --- a/src/main/java/com/salesforce/hbase/index/scanner/FilteredKeyValueScanner.java +++ b/src/main/java/com/salesforce/hbase/index/scanner/FilteredKeyValueScanner.java @@ -10,6 +10,8 @@ import org.apache.hadoop.hbase.regionserver.ExposedMemStore; import org.apache.hadoop.hbase.regionserver.KeyValueScanner; +import com.salesforce.hbase.index.covered.KeyValueStore; + /** * Combine a simplified version of the logic in the ScanQueryMatcher and the KeyValueScanner. We can * get away with this here because we are only concerned with a single {@link ExposedMemStore} for @@ -21,8 +23,8 @@ public class FilteredKeyValueScanner implements KeyValueScanner { private KeyValueScanner delegate; private Filter filter; - public FilteredKeyValueScanner(Filter filter, ExposedMemStore store) { - this(filter, store.getScanners().get(0)); + public FilteredKeyValueScanner(Filter filter, KeyValueStore store) { + this(filter, store.getScanner()); } private FilteredKeyValueScanner(Filter filter, KeyValueScanner delegate) { diff --git a/src/main/java/com/salesforce/hbase/index/scanner/ScannerBuilder.java b/src/main/java/com/salesforce/hbase/index/scanner/ScannerBuilder.java index 4190db36..15706775 100644 --- a/src/main/java/com/salesforce/hbase/index/scanner/ScannerBuilder.java +++ b/src/main/java/com/salesforce/hbase/index/scanner/ScannerBuilder.java @@ -13,10 +13,11 @@ import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.filter.QualifierFilter; -import org.apache.hadoop.hbase.regionserver.ExposedMemStore; import org.apache.hadoop.hbase.util.Bytes; import com.google.common.collect.Lists; +import com.salesforce.hbase.index.covered.KeyValueStore; +import com.salesforce.hbase.index.covered.data.IndexMemStore; import com.salesforce.hbase.index.covered.filter.ApplyAndFilterDeletesFilter; import com.salesforce.hbase.index.covered.filter.ColumnTrackingNextLargestTimestampFilter; import com.salesforce.hbase.index.covered.filter.MaxTimestampFilter; @@ -28,11 +29,11 @@ */ public class ScannerBuilder { - private ExposedMemStore memstore; + private KeyValueStore memstore; private Mutation update; - public ScannerBuilder(ExposedMemStore memstore, Mutation update) { + public ScannerBuilder(KeyValueStore memstore, Mutation update) { this.memstore = memstore; this.update = update; } diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/IndexKeyValueSkipListSet.java b/src/main/java/org/apache/hadoop/hbase/regionserver/IndexKeyValueSkipListSet.java new file mode 100644 index 00000000..6ad31043 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/IndexKeyValueSkipListSet.java @@ -0,0 +1,86 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package org.apache.hadoop.hbase.regionserver; + +import java.util.Comparator; +import java.util.concurrent.ConcurrentSkipListMap; + +import org.apache.hadoop.hbase.KeyValue; + +/** + * Like a {@link KeyValueSkipListSet}, but also exposes useful, atomic methods (e.g. + * {@link #putIfAbsent(KeyValue)}). + */ +public class IndexKeyValueSkipListSet extends KeyValueSkipListSet { + + // this is annoying that we need to keep this extra pointer around here, but its pretty minimal + // and means we don't need to change the HBase code. + private ConcurrentSkipListMap delegate; + + /** + * Create a new {@link IndexKeyValueSkipListSet} based on the passed comparator. + * @param comparator to use when comparing keyvalues. It is used both to determine sort order as + * well as object equality in the map. + * @return a map that uses the passed comparator + */ + public static IndexKeyValueSkipListSet create(Comparator comparator) { + ConcurrentSkipListMap delegate = + new ConcurrentSkipListMap(comparator); + IndexKeyValueSkipListSet ret = new IndexKeyValueSkipListSet(delegate); + return ret; + } + + /** + * @param delegate map to which to delegate all calls + */ + public IndexKeyValueSkipListSet(ConcurrentSkipListMap delegate) { + super(delegate); + this.delegate = delegate; + } + + /** + * Add the passed {@link KeyValue} to the set, only if one is not already set. This is equivalent + * to + * 

+   * if (!set.containsKey(key))
+   *   return set.put(key);
+   * else
+   *  return map.set(key);
+   *
+ * except that the action is performed atomically. + * @param kv {@link KeyValue} to add + * @return the previous value associated with the specified key, or null if there was no + * previously stored key + * @throws ClassCastException if the specified key cannot be compared with the keys currently in + * the map + * @throws NullPointerException if the specified key is null + */ + public KeyValue putIfAbsent(KeyValue kv) { + return this.delegate.putIfAbsent(kv, kv); + } +} \ No newline at end of file diff --git a/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java b/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java index fc2af974..7016bc00 100644 --- a/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java +++ b/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java @@ -47,7 +47,6 @@ import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import com.salesforce.hbase.index.covered.LocalTableState.PendingKeyValue; import com.salesforce.hbase.index.covered.data.LocalHBaseState; import com.salesforce.hbase.index.covered.data.LocalTable; import com.salesforce.hbase.index.covered.update.ColumnReference; @@ -139,7 +138,7 @@ public Boolean answer(InvocationOnMock invocation) throws Throwable { LocalHBaseState state = new LocalTable(env); LocalTableState table = new LocalTableState(env, state, m); // add the kvs from the mutation - KeyValue kv = new LocalTableState.PendingKeyValue(m.get(fam, qual).get(0)); + KeyValue kv = m.get(fam, qual).get(0); kv.setMemstoreTS(0); table.addPendingUpdates(kv); diff --git a/src/test/java/com/salesforce/hbase/index/covered/data/TestIndexMemStore.java b/src/test/java/com/salesforce/hbase/index/covered/data/TestIndexMemStore.java new file mode 100644 index 00000000..48627338 --- /dev/null +++ b/src/test/java/com/salesforce/hbase/index/covered/data/TestIndexMemStore.java @@ -0,0 +1,70 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index.covered.data; + +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.Type; +import org.apache.hadoop.hbase.regionserver.KeyValueScanner; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; + +public class TestIndexMemStore { + + private static final byte[] row = Bytes.toBytes("row"); + private static final byte[] family = Bytes.toBytes("family"); + private static final byte[] qual = Bytes.toBytes("qual"); + private static final byte[] val = Bytes.toBytes("val"); + private static final byte[] val2 = Bytes.toBytes("val2"); + + @Test + public void testCorrectOverwritting() throws Exception { + IndexMemStore store = new IndexMemStore(IndexMemStore.COMPARATOR); + long ts = 10; + KeyValue kv = new KeyValue(row, family, qual, ts, Type.Put, val); + kv.setMemstoreTS(2); + KeyValue kv2 = new KeyValue(row, family, qual, ts, Type.Put, val2); + kv2.setMemstoreTS(0); + store.add(kv, true); + // adding the exact same kv shouldn't change anything stored if not overwritting + store.add(kv2, false); + KeyValueScanner scanner = store.getScanner(); + KeyValue first = KeyValue.createFirstOnRow(row); + scanner.seek(first); + assertTrue("Overwrote kv when specifically not!", kv == scanner.next()); + scanner.close(); + + // now when we overwrite, we should get the newer one + store.add(kv2, true); + scanner = store.getScanner(); + scanner.seek(first); + assertTrue("Didn't overwrite kv when specifically requested!", kv2 == scanner.next()); + scanner.close(); + } +} \ No newline at end of file From 3637a87ba1b76202487015335809cff4e18d5ccd Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 20 Sep 2013 01:46:52 -0700 Subject: [PATCH 052/102] Fix select * and select cf.* ordering when index table used --- .../phoenix/compile/ColumnResolver.java | 5 +- .../compile/IndexStatementRewriter.java | 17 +- .../phoenix/compile/PostDDLCompiler.java | 6 +- .../phoenix/compile/ProjectionCompiler.java | 131 +++++-- .../phoenix/compile/QueryCompiler.java | 12 +- .../phoenix/compile/UpsertCompiler.java | 41 ++- .../expression/KeyValueColumnExpression.java | 18 +- .../expression/RowKeyColumnExpression.java | 2 +- .../phoenix/parse/ColumnParseNode.java | 12 +- ...Node.java => FamilyWildcardParseNode.java} | 34 +- .../phoenix/parse/NamedParseNode.java | 9 + .../phoenix/parse/ParseNodeFactory.java | 23 +- .../phoenix/parse/ParseNodeRewriter.java | 2 +- .../phoenix/parse/ParseNodeVisitor.java | 2 +- .../parse/TraverseAllParseNodeVisitor.java | 2 +- .../parse/TraverseNoParseNodeVisitor.java | 2 +- .../parse/UnsupportedAllParseNodeVisitor.java | 2 +- .../phoenix/parse/WildcardParseNode.java | 11 +- .../salesforce/phoenix/schema/ColumnRef.java | 5 +- .../salesforce/phoenix/util/IndexUtil.java | 5 + .../salesforce/phoenix/util/SchemaUtil.java | 15 +- .../end2end/index/BaseMutableIndexTest.java | 115 ++++++ ...IndexTest.java => ImmutableIndexTest.java} | 2 +- .../end2end/index/MutableIndexTest.java | 346 ++++++------------ .../end2end/index/MutableSaltedIndexTest.java | 195 ++++++++++ 25 files changed, 699 insertions(+), 315 deletions(-) rename src/main/java/com/salesforce/phoenix/parse/{FamilyParseNode.java => FamilyWildcardParseNode.java} (81%) create mode 100644 src/test/java/com/salesforce/phoenix/end2end/index/BaseMutableIndexTest.java rename src/test/java/com/salesforce/phoenix/end2end/index/{IndexTest.java => ImmutableIndexTest.java} (99%) create mode 100644 src/test/java/com/salesforce/phoenix/end2end/index/MutableSaltedIndexTest.java diff --git a/src/main/java/com/salesforce/phoenix/compile/ColumnResolver.java b/src/main/java/com/salesforce/phoenix/compile/ColumnResolver.java index 8db583f2..fd1d4e60 100644 --- a/src/main/java/com/salesforce/phoenix/compile/ColumnResolver.java +++ b/src/main/java/com/salesforce/phoenix/compile/ColumnResolver.java @@ -30,7 +30,10 @@ import java.sql.SQLException; import java.util.List; -import com.salesforce.phoenix.schema.*; +import com.salesforce.phoenix.schema.AmbiguousColumnException; +import com.salesforce.phoenix.schema.ColumnNotFoundException; +import com.salesforce.phoenix.schema.ColumnRef; +import com.salesforce.phoenix.schema.TableRef; diff --git a/src/main/java/com/salesforce/phoenix/compile/IndexStatementRewriter.java b/src/main/java/com/salesforce/phoenix/compile/IndexStatementRewriter.java index c3da06e5..f33ed745 100644 --- a/src/main/java/com/salesforce/phoenix/compile/IndexStatementRewriter.java +++ b/src/main/java/com/salesforce/phoenix/compile/IndexStatementRewriter.java @@ -3,9 +3,11 @@ import java.sql.SQLException; import com.salesforce.phoenix.parse.ColumnParseNode; +import com.salesforce.phoenix.parse.FamilyWildcardParseNode; import com.salesforce.phoenix.parse.ParseNode; import com.salesforce.phoenix.parse.ParseNodeRewriter; import com.salesforce.phoenix.parse.SelectStatement; +import com.salesforce.phoenix.parse.WildcardParseNode; import com.salesforce.phoenix.schema.ColumnRef; import com.salesforce.phoenix.util.IndexUtil; @@ -26,13 +28,22 @@ public IndexStatementRewriter(ColumnResolver resolver) { public static SelectStatement translate(SelectStatement statement, ColumnResolver resolver) throws SQLException { return rewrite(statement, new IndexStatementRewriter(resolver)); } - @Override public ParseNode visit(ColumnParseNode node) throws SQLException { ColumnRef ref = resolver.resolveColumn(node.getSchemaName(), node.getTableName(), node.getName()); - node = NODE_FACTORY.column(IndexUtil.getIndexColumnName(ref.getColumn())); - return node; + // Don't provide a TableName, as the column name for an index column will always be unique + return new ColumnParseNode(null, IndexUtil.getIndexColumnName(ref.getColumn()), node.toString()); + } + + @Override + public ParseNode visit(WildcardParseNode node) throws SQLException { + return WildcardParseNode.REWRITE_INSTANCE; } + @Override + public ParseNode visit(FamilyWildcardParseNode node) throws SQLException { + return new FamilyWildcardParseNode(node, true); + } + } diff --git a/src/main/java/com/salesforce/phoenix/compile/PostDDLCompiler.java b/src/main/java/com/salesforce/phoenix/compile/PostDDLCompiler.java index c941e9aa..ce04bf50 100644 --- a/src/main/java/com/salesforce/phoenix/compile/PostDDLCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/PostDDLCompiler.java @@ -48,7 +48,11 @@ import com.salesforce.phoenix.parse.SelectStatement; import com.salesforce.phoenix.query.QueryConstants; import com.salesforce.phoenix.query.Scanner; -import com.salesforce.phoenix.schema.*; +import com.salesforce.phoenix.schema.ColumnRef; +import com.salesforce.phoenix.schema.PColumn; +import com.salesforce.phoenix.schema.PColumnFamily; +import com.salesforce.phoenix.schema.PDataType; +import com.salesforce.phoenix.schema.TableRef; import com.salesforce.phoenix.schema.tuple.Tuple; import com.salesforce.phoenix.util.ScanUtil; diff --git a/src/main/java/com/salesforce/phoenix/compile/ProjectionCompiler.java b/src/main/java/com/salesforce/phoenix/compile/ProjectionCompiler.java index 9c595285..07c52a22 100644 --- a/src/main/java/com/salesforce/phoenix/compile/ProjectionCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/ProjectionCompiler.java @@ -28,11 +28,21 @@ package com.salesforce.phoenix.compile; import java.sql.SQLException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; +import java.util.Set; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; -import com.google.common.collect.*; +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.salesforce.phoenix.compile.GroupByCompiler.GroupBy; import com.salesforce.phoenix.coprocessor.GroupedAggregateRegionObserver; import com.salesforce.phoenix.expression.CoerceExpression; @@ -41,8 +51,24 @@ import com.salesforce.phoenix.expression.aggregator.ServerAggregators; import com.salesforce.phoenix.expression.function.SingleAggregateFunction; import com.salesforce.phoenix.expression.visitor.SingleAggregateFunctionVisitor; -import com.salesforce.phoenix.parse.*; -import com.salesforce.phoenix.schema.*; +import com.salesforce.phoenix.parse.AliasedNode; +import com.salesforce.phoenix.parse.BindParseNode; +import com.salesforce.phoenix.parse.ColumnParseNode; +import com.salesforce.phoenix.parse.FamilyWildcardParseNode; +import com.salesforce.phoenix.parse.ParseNode; +import com.salesforce.phoenix.parse.SelectStatement; +import com.salesforce.phoenix.parse.WildcardParseNode; +import com.salesforce.phoenix.schema.ArgumentTypeMismatchException; +import com.salesforce.phoenix.schema.ColumnNotFoundException; +import com.salesforce.phoenix.schema.ColumnRef; +import com.salesforce.phoenix.schema.PColumn; +import com.salesforce.phoenix.schema.PColumnFamily; +import com.salesforce.phoenix.schema.PDataType; +import com.salesforce.phoenix.schema.PTable; +import com.salesforce.phoenix.schema.PTableType; +import com.salesforce.phoenix.schema.RowKeySchema; +import com.salesforce.phoenix.schema.TableRef; +import com.salesforce.phoenix.util.IndexUtil; import com.salesforce.phoenix.util.SizedUtil; @@ -88,6 +114,62 @@ public static RowProjector compile(StatementContext context, SelectStatement sta return compile(context, statement, groupBy, null); } + private static void projectAllTableColumns(StatementContext context, TableRef tableRef, List projectedExpressions, List projectedColumns) throws SQLException { + PTable table = tableRef.getTable(); + for (int i = table.getBucketNum() == null ? 0 : 1; i < table.getColumns().size(); i++) { + ColumnRef ref = new ColumnRef(tableRef,i); + Expression expression = ref.newColumnExpression(); + projectedExpressions.add(expression); + projectedColumns.add(new ExpressionProjector(ref.getColumn().getName().getString(), table.getName().getString(), expression, false)); + } + } + + private static void projectAllIndexColumns(StatementContext context, TableRef tableRef, List projectedExpressions, List projectedColumns) throws SQLException { + PTable index = tableRef.getTable(); + PTable table = context.getConnection().getPMetaData().getTable(index.getParentName().getString()); + int tableOffset = table.getBucketNum() == null ? 0 : 1; + int indexOffset = index.getBucketNum() == null ? 0 : 1; + if (index.getColumns().size()-indexOffset != table.getColumns().size()-tableOffset) { + // We'll end up not using this by the optimizer, so just throw + throw new ColumnNotFoundException(WildcardParseNode.INSTANCE.toString()); + } + for (int i = tableOffset; i < table.getColumns().size(); i++) { + PColumn tableColumn = table.getColumns().get(i); + PColumn indexColumn = index.getColumn(IndexUtil.getIndexColumnName(tableColumn)); + ColumnRef ref = new ColumnRef(tableRef,indexColumn.getPosition()); + Expression expression = ref.newColumnExpression(); + projectedExpressions.add(expression); + ExpressionProjector projector = new ExpressionProjector(tableColumn.getName().getString(), table.getName().getString(), expression, false); + projectedColumns.add(projector); + } + } + + private static void projectTableColumnFamily(StatementContext context, String cfName, TableRef tableRef, List projectedExpressions, List projectedColumns) throws SQLException { + PTable table = tableRef.getTable(); + PColumnFamily pfamily = table.getColumnFamily(cfName); + for (PColumn column : pfamily.getColumns()) { + ColumnRef ref = new ColumnRef(tableRef, column.getPosition()); + Expression expression = ref.newColumnExpression(); + projectedExpressions.add(expression); + projectedColumns.add(new ExpressionProjector(column.getName().toString(), table.getName() + .getString(), expression, false)); + } + } + + private static void projectIndexColumnFamily(StatementContext context, String cfName, TableRef tableRef, List projectedExpressions, List projectedColumns) throws SQLException { + PTable index = tableRef.getTable(); + PTable table = context.getConnection().getPMetaData().getTable(index.getParentName().getString()); + PColumnFamily pfamily = table.getColumnFamily(cfName); + for (PColumn column : pfamily.getColumns()) { + PColumn indexColumn = index.getColumn(IndexUtil.getIndexColumnName(column)); + ColumnRef ref = new ColumnRef(tableRef, indexColumn.getPosition()); + Expression expression = ref.newColumnExpression(); + projectedExpressions.add(expression); + projectedColumns.add(new ExpressionProjector(column.getName().toString(), + table.getName().getString(), expression, false)); + } + } + /** * Builds the projection for the scan * @param context query context kept between compilation of different query clauses @@ -113,33 +195,32 @@ public static RowProjector compile(StatementContext context, SelectStatement sta // TODO: support cf.* expressions in projection to project all columns in a CF for (AliasedNode aliasedNode : aliasedNodes) { ParseNode node = aliasedNode.getNode(); - if (node == WildcardParseNode.INSTANCE) { + // TODO: visitor? + if (node instanceof WildcardParseNode) { if (statement.isAggregate()) { ExpressionCompiler.throwNonAggExpressionInAggException(node.toString()); } isWildcard = true; - for (int i = table.getBucketNum() == null ? 0 : 1; i < table.getColumns().size(); i++) { - ColumnRef ref = new ColumnRef(tableRef,i); - Expression expression = ref.newColumnExpression(); - projectedExpressions.add(expression); - projectedColumns.add(new ExpressionProjector(ref.getColumn().getName().getString(), table.getName().getString(), expression, false)); + if (tableRef.getTable().getType() == PTableType.INDEX && ((WildcardParseNode)node).isRewrite()) { + projectAllIndexColumns(context, tableRef, projectedExpressions, projectedColumns); + } else { + projectAllTableColumns(context, tableRef, projectedExpressions, projectedColumns); } - } else if (node instanceof FamilyParseNode){ + } else if (node instanceof FamilyWildcardParseNode){ // Project everything for SELECT cf.* - PColumnFamily pfamily = table.getColumnFamily(((FamilyParseNode) node).getFamilyName()); - // Delay projecting to scan, as when any other column in the column family gets - // added to the scan, it overwrites that we want to project the entire column - // family. Instead, we do the projection at the end. - // TODO: consider having a ScanUtil.addColumn and ScanUtil.addFamily to work - // around this, as this code depends on this function being the last place where - // columns are projected (which is currently true, but could change). - projectedFamilies.add(pfamily.getName().getBytes()); - for (PColumn column : pfamily.getColumns()) { - ColumnRef ref = new ColumnRef(tableRef,column.getPosition()); - Expression expression = ref.newColumnExpression(); - projectedExpressions.add(expression); - projectedColumns.add(new ExpressionProjector(column.getName().toString(), table.getName().getString(),expression, false)); - } + String cfName = ((FamilyWildcardParseNode) node).getName(); + // Delay projecting to scan, as when any other column in the column family gets + // added to the scan, it overwrites that we want to project the entire column + // family. Instead, we do the projection at the end. + // TODO: consider having a ScanUtil.addColumn and ScanUtil.addFamily to work + // around this, as this code depends on this function being the last place where + // columns are projected (which is currently true, but could change). + projectedFamilies.add(Bytes.toBytes(cfName)); + if (tableRef.getTable().getType() == PTableType.INDEX && ((FamilyWildcardParseNode)node).isRewrite()) { + projectIndexColumnFamily(context, cfName, tableRef, projectedExpressions, projectedColumns); + } else { + projectTableColumnFamily(context, cfName, tableRef, projectedExpressions, projectedColumns); + } } else { Expression expression = node.accept(selectVisitor); projectedExpressions.add(expression); diff --git a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java index 28dc9503..f5fe3632 100644 --- a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java @@ -36,7 +36,9 @@ import com.salesforce.phoenix.compile.GroupByCompiler.GroupBy; import com.salesforce.phoenix.compile.OrderByCompiler.OrderBy; -import com.salesforce.phoenix.execute.*; +import com.salesforce.phoenix.execute.AggregatePlan; +import com.salesforce.phoenix.execute.DegenerateQueryPlan; +import com.salesforce.phoenix.execute.ScanPlan; import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.iterate.ParallelIterators.ParallelIteratorFactory; import com.salesforce.phoenix.iterate.SpoolingResultIterator.SpoolingResultIteratorFactory; @@ -45,7 +47,13 @@ import com.salesforce.phoenix.parse.ParseNode; import com.salesforce.phoenix.parse.SelectStatement; import com.salesforce.phoenix.query.QueryConstants; -import com.salesforce.phoenix.schema.*; +import com.salesforce.phoenix.schema.AmbiguousColumnException; +import com.salesforce.phoenix.schema.ColumnNotFoundException; +import com.salesforce.phoenix.schema.PColumn; +import com.salesforce.phoenix.schema.PIndexState; +import com.salesforce.phoenix.schema.PTableType; +import com.salesforce.phoenix.schema.TableNotFoundException; +import com.salesforce.phoenix.schema.TableRef; diff --git a/src/main/java/com/salesforce/phoenix/compile/UpsertCompiler.java b/src/main/java/com/salesforce/phoenix/compile/UpsertCompiler.java index 8c22099b..80fe09f0 100644 --- a/src/main/java/com/salesforce/phoenix/compile/UpsertCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/UpsertCompiler.java @@ -27,8 +27,14 @@ ******************************************************************************/ package com.salesforce.phoenix.compile; -import java.sql.*; -import java.util.*; +import java.sql.ParameterMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Collections; +import java.util.List; +import java.util.Map; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Scan; @@ -47,13 +53,34 @@ import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.expression.LiteralExpression; import com.salesforce.phoenix.iterate.ParallelIterators.ParallelIteratorFactory; -import com.salesforce.phoenix.iterate.*; +import com.salesforce.phoenix.iterate.ResultIterator; +import com.salesforce.phoenix.iterate.SpoolingResultIterator; import com.salesforce.phoenix.iterate.SpoolingResultIterator.SpoolingResultIteratorFactory; -import com.salesforce.phoenix.jdbc.*; -import com.salesforce.phoenix.parse.*; -import com.salesforce.phoenix.query.*; +import com.salesforce.phoenix.jdbc.PhoenixConnection; +import com.salesforce.phoenix.jdbc.PhoenixResultSet; +import com.salesforce.phoenix.jdbc.PhoenixStatement; +import com.salesforce.phoenix.parse.BindParseNode; +import com.salesforce.phoenix.parse.ColumnName; +import com.salesforce.phoenix.parse.LiteralParseNode; +import com.salesforce.phoenix.parse.ParseNode; +import com.salesforce.phoenix.parse.SelectStatement; +import com.salesforce.phoenix.parse.UpsertStatement; +import com.salesforce.phoenix.query.ConnectionQueryServices; +import com.salesforce.phoenix.query.QueryServices; +import com.salesforce.phoenix.query.QueryServicesOptions; import com.salesforce.phoenix.query.Scanner; -import com.salesforce.phoenix.schema.*; +import com.salesforce.phoenix.schema.ColumnModifier; +import com.salesforce.phoenix.schema.ColumnRef; +import com.salesforce.phoenix.schema.ConstraintViolationException; +import com.salesforce.phoenix.schema.PColumn; +import com.salesforce.phoenix.schema.PColumnImpl; +import com.salesforce.phoenix.schema.PDataType; +import com.salesforce.phoenix.schema.PTable; +import com.salesforce.phoenix.schema.PTableImpl; +import com.salesforce.phoenix.schema.PTableType; +import com.salesforce.phoenix.schema.ReadOnlyTableException; +import com.salesforce.phoenix.schema.TableRef; +import com.salesforce.phoenix.schema.TypeMismatchException; import com.salesforce.phoenix.schema.tuple.Tuple; import com.salesforce.phoenix.util.SchemaUtil; diff --git a/src/main/java/com/salesforce/phoenix/expression/KeyValueColumnExpression.java b/src/main/java/com/salesforce/phoenix/expression/KeyValueColumnExpression.java index ccef8d5f..182ce045 100644 --- a/src/main/java/com/salesforce/phoenix/expression/KeyValueColumnExpression.java +++ b/src/main/java/com/salesforce/phoenix/expression/KeyValueColumnExpression.java @@ -27,7 +27,9 @@ ******************************************************************************/ package com.salesforce.phoenix.expression; -import java.io.*; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; import java.util.Arrays; import org.apache.hadoop.hbase.KeyValue; @@ -35,9 +37,9 @@ import org.apache.hadoop.hbase.util.Bytes; import com.salesforce.phoenix.expression.visitor.ExpressionVisitor; -import com.salesforce.phoenix.query.QueryConstants; import com.salesforce.phoenix.schema.PColumn; import com.salesforce.phoenix.schema.tuple.Tuple; +import com.salesforce.phoenix.util.SchemaUtil; /** @@ -50,12 +52,22 @@ public class KeyValueColumnExpression extends ColumnExpression { private byte[] cf; private byte[] cq; + private final String alias; // not serialized public KeyValueColumnExpression() { + this.alias = null; } public KeyValueColumnExpression(PColumn column) { super(column); + this.alias = null; + this.cf = column.getFamilyName().getBytes(); + this.cq = column.getName().getBytes(); + } + + public KeyValueColumnExpression(PColumn column, String alias) { + super(column); + this.alias = alias; this.cf = column.getFamilyName().getBytes(); this.cq = column.getName().getBytes(); } @@ -91,7 +103,7 @@ public boolean equals(Object obj) { @Override public String toString() { - return (Bytes.compareTo(cf, QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES) == 0 ? "" : (Bytes.toStringBinary(cf) + QueryConstants.NAME_SEPARATOR)) + Bytes.toStringBinary(cq); + return alias != null ? alias : SchemaUtil.getColumnDisplayName(cf, cq); } @Override diff --git a/src/main/java/com/salesforce/phoenix/expression/RowKeyColumnExpression.java b/src/main/java/com/salesforce/phoenix/expression/RowKeyColumnExpression.java index 2b2c9655..6be6e2cf 100644 --- a/src/main/java/com/salesforce/phoenix/expression/RowKeyColumnExpression.java +++ b/src/main/java/com/salesforce/phoenix/expression/RowKeyColumnExpression.java @@ -51,7 +51,7 @@ public class RowKeyColumnExpression extends ColumnExpression { private PDataType fromType; private RowKeyValueAccessor accessor; - private final String name; + protected final String name; public RowKeyColumnExpression() { name = null; // Only on client diff --git a/src/main/java/com/salesforce/phoenix/parse/ColumnParseNode.java b/src/main/java/com/salesforce/phoenix/parse/ColumnParseNode.java index 1d0f9d9a..0c34d674 100644 --- a/src/main/java/com/salesforce/phoenix/parse/ColumnParseNode.java +++ b/src/main/java/com/salesforce/phoenix/parse/ColumnParseNode.java @@ -28,19 +28,25 @@ public class ColumnParseNode extends NamedParseNode { private final TableName tableName; private final String fullName; + private final String alias; ColumnParseNode(String name) { - this(null, name); + this(null, name, null); } - public ColumnParseNode(TableName tableName, String name) { + public ColumnParseNode(TableName tableName, String name, String alias) { // Upper case here so our Maps can depend on this (and we don't have to upper case and create a string on every // lookup super(name); + this.alias = alias; this.tableName = tableName; fullName = tableName == null ? getName() : tableName.toString() + QueryConstants.NAME_SEPARATOR + getName(); } + public ColumnParseNode(TableName tableName, String name) { + this(tableName, name, null); + } + @Override public T accept(ParseNodeVisitor visitor) throws SQLException { return visitor.visit(this); @@ -60,7 +66,7 @@ public String getFullName() { @Override public String toString() { - return getName(); + return alias == null ? getName() : alias; } } diff --git a/src/main/java/com/salesforce/phoenix/parse/FamilyParseNode.java b/src/main/java/com/salesforce/phoenix/parse/FamilyWildcardParseNode.java similarity index 81% rename from src/main/java/com/salesforce/phoenix/parse/FamilyParseNode.java rename to src/main/java/com/salesforce/phoenix/parse/FamilyWildcardParseNode.java index aee2bd32..36094efb 100644 --- a/src/main/java/com/salesforce/phoenix/parse/FamilyParseNode.java +++ b/src/main/java/com/salesforce/phoenix/parse/FamilyWildcardParseNode.java @@ -38,26 +38,26 @@ * @since 1.2 */ -public class FamilyParseNode extends TerminalParseNode { - private String familyName = null; - - public FamilyParseNode(String familyName){ - this.familyName = familyName; - } - - public String getFamilyName(){ - return familyName; - } - +public class FamilyWildcardParseNode extends NamedParseNode { + private final boolean isRewrite; + + public FamilyWildcardParseNode(String familyName, boolean isRewrite){ + super(familyName); + this.isRewrite = isRewrite; + } + + public FamilyWildcardParseNode(FamilyWildcardParseNode familyName, boolean isRewrite){ + super(familyName); + this.isRewrite = isRewrite; + } + @Override public T accept(ParseNodeVisitor visitor) throws SQLException { return visitor.visit(this); } - - @Override - public String toString() { - return familyName+".*"; - } - + + public boolean isRewrite() { + return isRewrite; + } } diff --git a/src/main/java/com/salesforce/phoenix/parse/NamedParseNode.java b/src/main/java/com/salesforce/phoenix/parse/NamedParseNode.java index 244ca08e..66b5f02a 100644 --- a/src/main/java/com/salesforce/phoenix/parse/NamedParseNode.java +++ b/src/main/java/com/salesforce/phoenix/parse/NamedParseNode.java @@ -38,6 +38,10 @@ public abstract class NamedParseNode extends TerminalParseNode{ private final NamedNode namedNode; + NamedParseNode(NamedParseNode node) { + this.namedNode = node.namedNode; + } + NamedParseNode(String name) { this.namedNode = new NamedNode(name); } @@ -49,4 +53,9 @@ public String getName() { public boolean isCaseSensitive() { return namedNode.isCaseSensitive(); } + + @Override + public String toString() { + return getName(); + } } diff --git a/src/main/java/com/salesforce/phoenix/parse/ParseNodeFactory.java b/src/main/java/com/salesforce/phoenix/parse/ParseNodeFactory.java index 2daea98c..cee69acd 100644 --- a/src/main/java/com/salesforce/phoenix/parse/ParseNodeFactory.java +++ b/src/main/java/com/salesforce/phoenix/parse/ParseNodeFactory.java @@ -29,7 +29,11 @@ import java.lang.reflect.Constructor; import java.sql.SQLException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.util.Pair; @@ -39,11 +43,20 @@ import com.salesforce.phoenix.exception.UnknownFunctionException; import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.expression.ExpressionType; -import com.salesforce.phoenix.expression.function.*; +import com.salesforce.phoenix.expression.function.AvgAggregateFunction; +import com.salesforce.phoenix.expression.function.CountAggregateFunction; +import com.salesforce.phoenix.expression.function.CurrentDateFunction; +import com.salesforce.phoenix.expression.function.CurrentTimeFunction; +import com.salesforce.phoenix.expression.function.DistinctCountAggregateFunction; +import com.salesforce.phoenix.expression.function.FunctionExpression; import com.salesforce.phoenix.parse.FunctionParseNode.BuiltInFunction; import com.salesforce.phoenix.parse.FunctionParseNode.BuiltInFunctionInfo; import com.salesforce.phoenix.parse.JoinTableNode.JoinType; -import com.salesforce.phoenix.schema.*; +import com.salesforce.phoenix.schema.ColumnModifier; +import com.salesforce.phoenix.schema.PDataType; +import com.salesforce.phoenix.schema.PIndexState; +import com.salesforce.phoenix.schema.PTableType; +import com.salesforce.phoenix.schema.TypeMismatchException; import com.salesforce.phoenix.util.SchemaUtil; @@ -198,8 +211,8 @@ public AndParseNode and(List children) { return new AndParseNode(children); } - public FamilyParseNode family(String familyName){ - return new FamilyParseNode(familyName); + public FamilyWildcardParseNode family(String familyName){ + return new FamilyWildcardParseNode(familyName, false); } public WildcardParseNode wildcard() { diff --git a/src/main/java/com/salesforce/phoenix/parse/ParseNodeRewriter.java b/src/main/java/com/salesforce/phoenix/parse/ParseNodeRewriter.java index a9083917..e00cacb6 100644 --- a/src/main/java/com/salesforce/phoenix/parse/ParseNodeRewriter.java +++ b/src/main/java/com/salesforce/phoenix/parse/ParseNodeRewriter.java @@ -321,7 +321,7 @@ public ParseNode visit(WildcardParseNode node) throws SQLException { } @Override - public ParseNode visit(FamilyParseNode node) throws SQLException { + public ParseNode visit(FamilyWildcardParseNode node) throws SQLException { return node; } diff --git a/src/main/java/com/salesforce/phoenix/parse/ParseNodeVisitor.java b/src/main/java/com/salesforce/phoenix/parse/ParseNodeVisitor.java index 515b014e..7ea8e672 100644 --- a/src/main/java/com/salesforce/phoenix/parse/ParseNodeVisitor.java +++ b/src/main/java/com/salesforce/phoenix/parse/ParseNodeVisitor.java @@ -95,7 +95,7 @@ public interface ParseNodeVisitor { public E visit(LiteralParseNode node) throws SQLException; public E visit(BindParseNode node) throws SQLException; public E visit(WildcardParseNode node) throws SQLException; - public E visit(FamilyParseNode node) throws SQLException; + public E visit(FamilyWildcardParseNode node) throws SQLException; public E visit(ParseNode node) throws SQLException; public boolean visitEnter(StringConcatParseNode node) throws SQLException; diff --git a/src/main/java/com/salesforce/phoenix/parse/TraverseAllParseNodeVisitor.java b/src/main/java/com/salesforce/phoenix/parse/TraverseAllParseNodeVisitor.java index e73f1761..e9b9eb0f 100644 --- a/src/main/java/com/salesforce/phoenix/parse/TraverseAllParseNodeVisitor.java +++ b/src/main/java/com/salesforce/phoenix/parse/TraverseAllParseNodeVisitor.java @@ -140,7 +140,7 @@ public T visit(WildcardParseNode node) throws SQLException { } @Override - public T visit(FamilyParseNode node) throws SQLException { + public T visit(FamilyWildcardParseNode node) throws SQLException { return null; } diff --git a/src/main/java/com/salesforce/phoenix/parse/TraverseNoParseNodeVisitor.java b/src/main/java/com/salesforce/phoenix/parse/TraverseNoParseNodeVisitor.java index 5211479e..0dc75faf 100644 --- a/src/main/java/com/salesforce/phoenix/parse/TraverseNoParseNodeVisitor.java +++ b/src/main/java/com/salesforce/phoenix/parse/TraverseNoParseNodeVisitor.java @@ -140,7 +140,7 @@ public T visit(WildcardParseNode node) throws SQLException { } @Override - public T visit(FamilyParseNode node) throws SQLException { + public T visit(FamilyWildcardParseNode node) throws SQLException { return null; } diff --git a/src/main/java/com/salesforce/phoenix/parse/UnsupportedAllParseNodeVisitor.java b/src/main/java/com/salesforce/phoenix/parse/UnsupportedAllParseNodeVisitor.java index b3285418..372f6a84 100644 --- a/src/main/java/com/salesforce/phoenix/parse/UnsupportedAllParseNodeVisitor.java +++ b/src/main/java/com/salesforce/phoenix/parse/UnsupportedAllParseNodeVisitor.java @@ -65,7 +65,7 @@ public E visit(WildcardParseNode node) throws SQLException { } @Override - public E visit(FamilyParseNode node) throws SQLException { + public E visit(FamilyWildcardParseNode node) throws SQLException { throw new SQLFeatureNotSupportedException(node.toString()); } diff --git a/src/main/java/com/salesforce/phoenix/parse/WildcardParseNode.java b/src/main/java/com/salesforce/phoenix/parse/WildcardParseNode.java index ccbd688a..dc165079 100644 --- a/src/main/java/com/salesforce/phoenix/parse/WildcardParseNode.java +++ b/src/main/java/com/salesforce/phoenix/parse/WildcardParseNode.java @@ -39,9 +39,12 @@ * @since 0.1 */ public class WildcardParseNode extends TerminalParseNode { - public static final WildcardParseNode INSTANCE = new WildcardParseNode(); + private final boolean isRewrite; + public static final WildcardParseNode INSTANCE = new WildcardParseNode(false); + public static final WildcardParseNode REWRITE_INSTANCE = new WildcardParseNode(true); - private WildcardParseNode() { + private WildcardParseNode(boolean isRewrite) { + this.isRewrite = isRewrite; } @Override @@ -52,6 +55,10 @@ public T accept(ParseNodeVisitor visitor) throws SQLException { @Override public String toString() { return "*"; + } + + public boolean isRewrite() { + return isRewrite; } } diff --git a/src/main/java/com/salesforce/phoenix/schema/ColumnRef.java b/src/main/java/com/salesforce/phoenix/schema/ColumnRef.java index f0c27e25..68153939 100644 --- a/src/main/java/com/salesforce/phoenix/schema/ColumnRef.java +++ b/src/main/java/com/salesforce/phoenix/schema/ColumnRef.java @@ -96,7 +96,10 @@ public ColumnExpression newColumnExpression() throws SQLException { if (SchemaUtil.isPKColumn(this.getColumn())) { return new RowKeyColumnExpression(getColumn(), new RowKeyValueAccessor(this.getTable().getPKColumns(), pkSlotPosition)); } else { - return new KeyValueColumnExpression(getColumn()); + PColumn column = getColumn(); + boolean isIndex = tableRef.getTable().getType() == PTableType.INDEX; + String alias = isIndex ? SchemaUtil.getColumnDisplayName(column.getName().getBytes()) : null; + return new KeyValueColumnExpression(column, alias); } } diff --git a/src/main/java/com/salesforce/phoenix/util/IndexUtil.java b/src/main/java/com/salesforce/phoenix/util/IndexUtil.java index c1a71510..7e3a86c8 100644 --- a/src/main/java/com/salesforce/phoenix/util/IndexUtil.java +++ b/src/main/java/com/salesforce/phoenix/util/IndexUtil.java @@ -86,6 +86,11 @@ public static PDataType getIndexColumnDataType(boolean isNullable, PDataType dat return null; } + + public static String getDataColumnName(String name) { + return name.substring(name.indexOf(INDEX_COLUMN_NAME_SEP)); + } + public static String getIndexColumnName(String dataColumnFamilyName, String dataColumnName) { return (dataColumnFamilyName == null ? "" : dataColumnFamilyName) + INDEX_COLUMN_NAME_SEP + dataColumnName; } diff --git a/src/main/java/com/salesforce/phoenix/util/SchemaUtil.java b/src/main/java/com/salesforce/phoenix/util/SchemaUtil.java index 08eaa50f..39a909e7 100644 --- a/src/main/java/com/salesforce/phoenix/util/SchemaUtil.java +++ b/src/main/java/com/salesforce/phoenix/util/SchemaUtil.java @@ -268,7 +268,15 @@ private static String getName(String optionalQualifier, String name) { } public static String getTableName(byte[] schemaName, byte[] tableName) { - return Bytes.toString(getTableNameAsBytes(schemaName,tableName)); + return getName(schemaName, tableName); + } + + public static String getColumnDisplayName(byte[] cf, byte[] cq) { + return getName(Bytes.compareTo(cf, QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES) == 0 ? ByteUtil.EMPTY_BYTE_ARRAY : cf, cq); + } + + public static String getColumnDisplayName(byte[] cq) { + return getName(ByteUtil.EMPTY_BYTE_ARRAY, cq); } public static String getMetaDataEntityName(String schemaName, String tableName, String familyName, String columnName) { @@ -306,6 +314,10 @@ private static byte[] getNameAsBytes(byte[] nameOne, byte[] nameTwo) { } } + public static String getName(byte[] nameOne, byte[] nameTwo) { + return Bytes.toString(getNameAsBytes(nameOne,nameTwo)); + } + public static int getUnpaddedCharLength(byte[] b, int offset, int length, ColumnModifier columnModifier) { int i = offset + length -1; // If bytes are inverted, we need to invert the byte we're looking for too @@ -416,6 +428,7 @@ public static boolean isMetaTable(String schemaName, String tableName) { // Given the splits and the rowKeySchema, find out the keys that public static byte[][] processSplits(byte[][] splits, List pkColumns, Integer saltBucketNum, boolean defaultRowKeyOrder) throws SQLException { + // FIXME: shouldn't this return if splits.length == 0? if (splits == null) return null; // We do not accept user specified splits if the table is salted and we specify defaultRowKeyOrder. In this case, // throw an exception. diff --git a/src/test/java/com/salesforce/phoenix/end2end/index/BaseMutableIndexTest.java b/src/test/java/com/salesforce/phoenix/end2end/index/BaseMutableIndexTest.java new file mode 100644 index 00000000..298ec94e --- /dev/null +++ b/src/test/java/com/salesforce/phoenix/end2end/index/BaseMutableIndexTest.java @@ -0,0 +1,115 @@ +package com.salesforce.phoenix.end2end.index; + +import static com.salesforce.phoenix.util.TestUtil.TEST_PROPERTIES; + +import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.Properties; + +import com.salesforce.phoenix.end2end.BaseHBaseManagedTimeTest; +import com.salesforce.phoenix.util.SchemaUtil; + +public class BaseMutableIndexTest extends BaseHBaseManagedTimeTest { + public static final String SCHEMA_NAME = ""; + public static final String DATA_TABLE_NAME = "T"; + public static final String INDEX_TABLE_NAME = "I"; + public static final String DATA_TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME, "T"); + public static final String INDEX_TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME, "I"); + + public BaseMutableIndexTest() { + } + + protected static void createTestTable() throws SQLException { + String ddl = "create table if not exists " + DATA_TABLE_FULL_NAME + "(" + + " varchar_pk VARCHAR NOT NULL, " + + " char_pk CHAR(5) NOT NULL, " + + " int_pk INTEGER NOT NULL, "+ + " long_pk BIGINT NOT NULL, " + + " decimal_pk DECIMAL(31, 10) NOT NULL, " + + " a.varchar_col1 VARCHAR, " + + " a.char_col1 CHAR(5), " + + " a.int_col1 INTEGER, " + + " a.long_col1 BIGINT, " + + " a.decimal_col1 DECIMAL(31, 10), " + + " b.varchar_col2 VARCHAR, " + + " b.char_col2 CHAR(5), " + + " b.int_col2 INTEGER, " + + " b.long_col2 BIGINT, " + + " b.decimal_col2 DECIMAL(31, 10) " + + " CONSTRAINT pk PRIMARY KEY (varchar_pk, char_pk, int_pk, long_pk DESC, decimal_pk))"; + Properties props = new Properties(TEST_PROPERTIES); + Connection conn = DriverManager.getConnection(getUrl(), props); + conn.createStatement().execute(ddl); + conn.close(); + } + + // Populate the test table with data. + protected static void populateTestTable() throws SQLException { + Properties props = new Properties(TEST_PROPERTIES); + Connection conn = DriverManager.getConnection(getUrl(), props); + try { + String upsert = "UPSERT INTO " + DATA_TABLE_FULL_NAME + + " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + PreparedStatement stmt = conn.prepareStatement(upsert); + stmt.setString(1, "varchar1"); + stmt.setString(2, "char1"); + stmt.setInt(3, 1); + stmt.setLong(4, 1L); + stmt.setBigDecimal(5, new BigDecimal(1.0)); + stmt.setString(6, "varchar_a"); + stmt.setString(7, "chara"); + stmt.setInt(8, 2); + stmt.setLong(9, 2L); + stmt.setBigDecimal(10, new BigDecimal(2.0)); + stmt.setString(11, "varchar_b"); + stmt.setString(12, "charb"); + stmt.setInt(13, 3); + stmt.setLong(14, 3L); + stmt.setBigDecimal(15, new BigDecimal(3.0)); + stmt.executeUpdate(); + + stmt.setString(1, "varchar2"); + stmt.setString(2, "char2"); + stmt.setInt(3, 2); + stmt.setLong(4, 2L); + stmt.setBigDecimal(5, new BigDecimal(2.0)); + stmt.setString(6, "varchar_a"); + stmt.setString(7, "chara"); + stmt.setInt(8, 3); + stmt.setLong(9, 3L); + stmt.setBigDecimal(10, new BigDecimal(3.0)); + stmt.setString(11, "varchar_b"); + stmt.setString(12, "charb"); + stmt.setInt(13, 4); + stmt.setLong(14, 4L); + stmt.setBigDecimal(15, new BigDecimal(4.0)); + stmt.executeUpdate(); + + stmt.setString(1, "varchar3"); + stmt.setString(2, "char3"); + stmt.setInt(3, 3); + stmt.setLong(4, 3L); + stmt.setBigDecimal(5, new BigDecimal(3.0)); + stmt.setString(6, "varchar_a"); + stmt.setString(7, "chara"); + stmt.setInt(8, 4); + stmt.setLong(9, 4L); + stmt.setBigDecimal(10, new BigDecimal(4.0)); + stmt.setString(11, "varchar_b"); + stmt.setString(12, "charb"); + stmt.setInt(13, 5); + stmt.setLong(14, 5L); + stmt.setBigDecimal(15, new BigDecimal(5.0)); + stmt.executeUpdate(); + + conn.commit(); + } finally { + conn.close(); + } + } + + +} diff --git a/src/test/java/com/salesforce/phoenix/end2end/index/IndexTest.java b/src/test/java/com/salesforce/phoenix/end2end/index/ImmutableIndexTest.java similarity index 99% rename from src/test/java/com/salesforce/phoenix/end2end/index/IndexTest.java rename to src/test/java/com/salesforce/phoenix/end2end/index/ImmutableIndexTest.java index 389f1721..b76b18d0 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/index/IndexTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/index/ImmutableIndexTest.java @@ -33,7 +33,7 @@ import com.salesforce.phoenix.util.SchemaUtil; -public class IndexTest extends BaseHBaseManagedTimeTest{ +public class ImmutableIndexTest extends BaseHBaseManagedTimeTest{ private static final int TABLE_SPLITS = 3; private static final int INDEX_SPLITS = 4; private static final byte[] DATA_TABLE_FULL_NAME = Bytes.toBytes(SchemaUtil.getTableName(null, "T")); diff --git a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java index ea9c1759..f635e308 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java @@ -5,12 +5,10 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.math.BigDecimal; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.sql.SQLException; import java.util.Map; import java.util.Properties; @@ -21,25 +19,15 @@ import org.junit.Test; import com.google.common.collect.Maps; -import com.salesforce.phoenix.end2end.BaseHBaseManagedTimeTest; import com.salesforce.phoenix.jdbc.PhoenixConnection; import com.salesforce.phoenix.query.ConnectionQueryServices; import com.salesforce.phoenix.query.QueryServices; import com.salesforce.phoenix.util.QueryUtil; import com.salesforce.phoenix.util.ReadOnlyProps; -import com.salesforce.phoenix.util.SchemaUtil; -public class MutableIndexTest extends BaseHBaseManagedTimeTest{ - private static final int TABLE_SPLITS = 3; - private static final int INDEX_SPLITS = 4; - public static final String SCHEMA_NAME = ""; - public static final String DATA_TABLE_NAME = "T"; - public static final String INDEX_TABLE_NAME = "I"; - private static final String DATA_TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME, "T"); - private static final String INDEX_TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME, "I"); - - @BeforeClass +public class MutableIndexTest extends BaseMutableIndexTest { + @BeforeClass // FIXME: We should not need to do this public static void doSetup() throws Exception { Map props = Maps.newHashMapWithExpectedSize(1); // Don't cache meta information for this test because the splits change between tests @@ -47,97 +35,8 @@ public static void doSetup() throws Exception { // Must update config before starting server startServer(getUrl(), new ReadOnlyProps(props.entrySet().iterator())); } - - private static void createTestTable() throws SQLException { - String ddl = "create table if not exists " + DATA_TABLE_FULL_NAME + "(" + - " varchar_pk VARCHAR NOT NULL, " + - " char_pk CHAR(5) NOT NULL, " + - " int_pk INTEGER NOT NULL, "+ - " long_pk BIGINT NOT NULL, " + - " decimal_pk DECIMAL(31, 10) NOT NULL, " + - " a.varchar_col1 VARCHAR, " + - " a.char_col1 CHAR(5), " + - " a.int_col1 INTEGER, " + - " a.long_col1 BIGINT, " + - " a.decimal_col1 DECIMAL(31, 10), " + - " b.varchar_col2 VARCHAR, " + - " b.char_col2 CHAR(5), " + - " b.int_col2 INTEGER, " + - " b.long_col2 BIGINT, " + - " b.decimal_col2 DECIMAL(31, 10) " + - " CONSTRAINT pk PRIMARY KEY (varchar_pk, char_pk, int_pk, long_pk DESC, decimal_pk))"; - Properties props = new Properties(TEST_PROPERTIES); - Connection conn = DriverManager.getConnection(getUrl(), props); - conn.createStatement().execute(ddl); - conn.close(); - } - - // Populate the test table with data. - private static void populateTestTable() throws SQLException { - Properties props = new Properties(TEST_PROPERTIES); - Connection conn = DriverManager.getConnection(getUrl(), props); - try { - String upsert = "UPSERT INTO " + DATA_TABLE_FULL_NAME - + " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; - PreparedStatement stmt = conn.prepareStatement(upsert); - stmt.setString(1, "varchar1"); - stmt.setString(2, "char1"); - stmt.setInt(3, 1); - stmt.setLong(4, 1L); - stmt.setBigDecimal(5, new BigDecimal(1.0)); - stmt.setString(6, "varchar_a"); - stmt.setString(7, "chara"); - stmt.setInt(8, 2); - stmt.setLong(9, 2L); - stmt.setBigDecimal(10, new BigDecimal(2.0)); - stmt.setString(11, "varchar_b"); - stmt.setString(12, "charb"); - stmt.setInt(13, 3); - stmt.setLong(14, 3L); - stmt.setBigDecimal(15, new BigDecimal(3.0)); - stmt.executeUpdate(); - - stmt.setString(1, "varchar2"); - stmt.setString(2, "char2"); - stmt.setInt(3, 2); - stmt.setLong(4, 2L); - stmt.setBigDecimal(5, new BigDecimal(2.0)); - stmt.setString(6, "varchar_a"); - stmt.setString(7, "chara"); - stmt.setInt(8, 3); - stmt.setLong(9, 3L); - stmt.setBigDecimal(10, new BigDecimal(3.0)); - stmt.setString(11, "varchar_b"); - stmt.setString(12, "charb"); - stmt.setInt(13, 4); - stmt.setLong(14, 4L); - stmt.setBigDecimal(15, new BigDecimal(4.0)); - stmt.executeUpdate(); - - stmt.setString(1, "varchar3"); - stmt.setString(2, "char3"); - stmt.setInt(3, 3); - stmt.setLong(4, 3L); - stmt.setBigDecimal(5, new BigDecimal(3.0)); - stmt.setString(6, "varchar_a"); - stmt.setString(7, "chara"); - stmt.setInt(8, 4); - stmt.setLong(9, 4L); - stmt.setBigDecimal(10, new BigDecimal(4.0)); - stmt.setString(11, "varchar_b"); - stmt.setString(12, "charb"); - stmt.setInt(13, 5); - stmt.setLong(14, 5L); - stmt.setBigDecimal(15, new BigDecimal(5.0)); - stmt.executeUpdate(); - - conn.commit(); - } finally { - conn.close(); - } - } - - @Before + + @Before // FIXME: We should not need to destroy the tables between these tests public void destroyTables() throws Exception { // Physically delete HBase table so that splits occur as expected for each test Properties props = new Properties(TEST_PROPERTIES); @@ -159,137 +58,6 @@ public void destroyTables() throws Exception { } } - @Test - public void testMutableTableIndexMaintanenceSaltedSalted() throws Exception { - testMutableTableIndexMaintanence(TABLE_SPLITS, INDEX_SPLITS); - } - - @Test - public void testMutableTableIndexMaintanenceSalted() throws Exception { - testMutableTableIndexMaintanence(null, INDEX_SPLITS); - } - - @Test - public void testMutableTableIndexMaintanenceUnsalted() throws Exception { - testMutableTableIndexMaintanence(null, null); - } - - private void testMutableTableIndexMaintanence(Integer tableSaltBuckets, Integer indexSaltBuckets) throws Exception { - String query; - ResultSet rs; - - Properties props = new Properties(TEST_PROPERTIES); - Connection conn = DriverManager.getConnection(getUrl(), props); - conn.setAutoCommit(false); - conn.createStatement().execute("CREATE TABLE " + DATA_TABLE_FULL_NAME + " (k VARCHAR NOT NULL PRIMARY KEY, v VARCHAR) " + (tableSaltBuckets == null ? "" : " SALT_BUCKETS=" + tableSaltBuckets)); - query = "SELECT * FROM " + DATA_TABLE_FULL_NAME; - rs = conn.createStatement().executeQuery(query); - assertFalse(rs.next()); - - conn.createStatement().execute("CREATE INDEX " + INDEX_TABLE_NAME + " ON " + DATA_TABLE_FULL_NAME + " (v DESC)" + (indexSaltBuckets == null ? "" : " SALT_BUCKETS=" + indexSaltBuckets)); - query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME; - rs = conn.createStatement().executeQuery(query); - assertFalse(rs.next()); - - PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?)"); - stmt.setString(1,"a"); - stmt.setString(2, "x"); - stmt.execute(); - stmt.setString(1,"b"); - stmt.setString(2, "y"); - stmt.execute(); - conn.commit(); - - query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME; - rs = conn.createStatement().executeQuery(query); - assertTrue(rs.next()); - assertEquals("y",rs.getString(1)); - assertEquals("b",rs.getString(2)); - assertTrue(rs.next()); - assertEquals("x",rs.getString(1)); - assertEquals("a",rs.getString(2)); - assertFalse(rs.next()); - - query = "SELECT k,v FROM " + DATA_TABLE_FULL_NAME + " WHERE v = 'y'"; - rs = conn.createStatement().executeQuery(query); - assertTrue(rs.next()); - assertEquals("b",rs.getString(1)); - assertEquals("y",rs.getString(2)); - assertFalse(rs.next()); - - String expectedPlan; - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - expectedPlan = indexSaltBuckets == null ? - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + INDEX_TABLE_FULL_NAME + " 'y'" : - ("CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 KEYS OVER " + INDEX_TABLE_FULL_NAME + " 0...3,'y'\n" + - "CLIENT MERGE SORT"); - assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); - - // Will use index, so rows returned in DESC order. - // This is not a bug, though, because we can - // return in any order. - query = "SELECT k,v FROM " + DATA_TABLE_FULL_NAME + " WHERE v >= 'x'"; - rs = conn.createStatement().executeQuery(query); - assertTrue(rs.next()); - assertEquals("b",rs.getString(1)); - assertEquals("y",rs.getString(2)); - assertTrue(rs.next()); - assertEquals("a",rs.getString(1)); - assertEquals("x",rs.getString(2)); - assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - expectedPlan = indexSaltBuckets == null ? - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + INDEX_TABLE_FULL_NAME + " (*-'x']" : - ("CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 RANGES OVER " + INDEX_TABLE_FULL_NAME + " 0...3,(*-'x']\n" + - "CLIENT MERGE SORT"); - assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); - - // Will still use index, since there's no LIMIT clause - query = "SELECT k,v FROM " + DATA_TABLE_FULL_NAME + " WHERE v >= 'x' ORDER BY k"; - rs = conn.createStatement().executeQuery(query); - assertTrue(rs.next()); - assertEquals("a",rs.getString(1)); - assertEquals("x",rs.getString(2)); - assertTrue(rs.next()); - assertEquals("b",rs.getString(1)); - assertEquals("y",rs.getString(2)); - assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - // Turns into an ORDER BY, which could be bad if lots of data is - // being returned. Without stats we don't know. The alternative - // would be a full table scan. - expectedPlan = indexSaltBuckets == null ? - ("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + INDEX_TABLE_FULL_NAME + " (*-'x']\n" + - " SERVER TOP -1 ROWS SORTED BY [:K]\n" + - "CLIENT MERGE SORT") : - ("CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 RANGES OVER " + INDEX_TABLE_FULL_NAME + " 0...3,(*-'x']\n" + - " SERVER TOP -1 ROWS SORTED BY [:K]\n" + - "CLIENT MERGE SORT"); - assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); - - // Will use data table now, since there's a LIMIT clause and - // we're able to optimize out the ORDER BY, unless the data - // table is salted. - query = "SELECT k,v FROM " + DATA_TABLE_FULL_NAME + " WHERE v >= 'x' ORDER BY k LIMIT 2"; - rs = conn.createStatement().executeQuery(query); - assertTrue(rs.next()); - assertEquals("a",rs.getString(1)); - assertEquals("x",rs.getString(2)); - assertTrue(rs.next()); - assertEquals("b",rs.getString(1)); - assertEquals("y",rs.getString(2)); - assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - expectedPlan = tableSaltBuckets == null ? - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + DATA_TABLE_FULL_NAME + "\n" + - " SERVER FILTER BY V >= 'x'\n" + - " SERVER 2 ROW LIMIT\n" + - "CLIENT 2 ROW LIMIT" : - "CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 RANGES OVER " + INDEX_TABLE_FULL_NAME + " 0...3,(*-'x']\n" + - " SERVER TOP 2 ROWS SORTED BY [:K]\n" + - "CLIENT MERGE SORT"; - assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); - } @Test public void testIndexWithNullableFixedWithCols() throws Exception { @@ -341,7 +109,8 @@ public void testCoveredColumnUpdates() throws Exception { String query = "SELECT char_col1, int_col1, long_col2 from " + DATA_TABLE_FULL_NAME; ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + INDEX_TABLE_FULL_NAME, QueryUtil.getExplainPlan(rs)); + // FIXME: Why is this a 4 way parallel scan while the data table is a 1 way scan? + assertEquals("CLIENT PARALLEL 4-WAY FULL SCAN OVER " + INDEX_TABLE_FULL_NAME, QueryUtil.getExplainPlan(rs)); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -407,4 +176,107 @@ public void testCoveredColumnUpdates() throws Exception { conn.close(); } } + + @Test + public void testSelectAll() throws Exception { + String query; + ResultSet rs; + + Properties props = new Properties(TEST_PROPERTIES); + Connection conn = DriverManager.getConnection(getUrl(), props); + conn.setAutoCommit(false); + conn.createStatement().execute("CREATE TABLE " + DATA_TABLE_FULL_NAME + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)"); + query = "SELECT * FROM " + DATA_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertFalse(rs.next()); + + conn.createStatement().execute("CREATE INDEX " + INDEX_TABLE_NAME + " ON " + DATA_TABLE_FULL_NAME + " (v2 DESC) INCLUDE (v1)"); + query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertFalse(rs.next()); + + PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?)"); + stmt.setString(1,"a"); + stmt.setString(2, "x"); + stmt.setString(3, "1"); + stmt.execute(); + stmt.setString(1,"b"); + stmt.setString(2, "y"); + stmt.setString(3, "2"); + stmt.execute(); + conn.commit(); + + query = "SELECT * FROM " + DATA_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery("EXPLAIN " + query); + assertEquals("CLIENT PARALLEL 4-WAY FULL SCAN OVER " + INDEX_TABLE_FULL_NAME, QueryUtil.getExplainPlan(rs)); + + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("b",rs.getString(1)); + assertEquals("y",rs.getString(2)); + assertEquals("2",rs.getString(3)); + assertEquals("b",rs.getString("k")); + assertEquals("y",rs.getString("v1")); + assertEquals("2",rs.getString("v2")); + assertTrue(rs.next()); + assertEquals("a",rs.getString(1)); + assertEquals("x",rs.getString(2)); + assertEquals("1",rs.getString(3)); + assertEquals("a",rs.getString("k")); + assertEquals("x",rs.getString("v1")); + assertEquals("1",rs.getString("v2")); + assertFalse(rs.next()); + } + + @Test + public void testSelectCF() throws Exception { + String query; + ResultSet rs; + + Properties props = new Properties(TEST_PROPERTIES); + Connection conn = DriverManager.getConnection(getUrl(), props); + conn.setAutoCommit(false); + conn.createStatement().execute("CREATE TABLE " + DATA_TABLE_FULL_NAME + " (k VARCHAR NOT NULL PRIMARY KEY, a.v1 VARCHAR, a.v2 VARCHAR, b.v1 VARCHAR) "); + query = "SELECT * FROM " + DATA_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertFalse(rs.next()); + + conn.createStatement().execute("CREATE INDEX " + INDEX_TABLE_NAME + " ON " + DATA_TABLE_FULL_NAME + " (v2 DESC) INCLUDE (a.v1)"); + query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertFalse(rs.next()); + + PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?,?)"); + stmt.setString(1,"a"); + stmt.setString(2, "x"); + stmt.setString(3, "1"); + stmt.setString(4, "A"); + stmt.execute(); + stmt.setString(1,"b"); + stmt.setString(2, "y"); + stmt.setString(3, "2"); + stmt.setString(4, "B"); + stmt.execute(); + conn.commit(); + + query = "SELECT * FROM " + DATA_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery("EXPLAIN " + query); + assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + DATA_TABLE_FULL_NAME, QueryUtil.getExplainPlan(rs)); + + query = "SELECT a.* FROM " + DATA_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery("EXPLAIN " + query); + assertEquals("CLIENT PARALLEL 4-WAY FULL SCAN OVER " + INDEX_TABLE_FULL_NAME, QueryUtil.getExplainPlan(rs)); + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("y",rs.getString(1)); + assertEquals("2",rs.getString(2)); + assertEquals("y",rs.getString("v1")); + assertEquals("2",rs.getString("v2")); + assertTrue(rs.next()); + assertEquals("x",rs.getString(1)); + assertEquals("1",rs.getString(2)); + assertEquals("x",rs.getString("v1")); + assertEquals("1",rs.getString("v2")); + assertFalse(rs.next()); + } } diff --git a/src/test/java/com/salesforce/phoenix/end2end/index/MutableSaltedIndexTest.java b/src/test/java/com/salesforce/phoenix/end2end/index/MutableSaltedIndexTest.java new file mode 100644 index 00000000..f36d49a6 --- /dev/null +++ b/src/test/java/com/salesforce/phoenix/end2end/index/MutableSaltedIndexTest.java @@ -0,0 +1,195 @@ +package com.salesforce.phoenix.end2end.index; + +import static com.salesforce.phoenix.util.TestUtil.TEST_PROPERTIES; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.Map; +import java.util.Properties; + +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.google.common.collect.Maps; +import com.salesforce.phoenix.jdbc.PhoenixConnection; +import com.salesforce.phoenix.query.ConnectionQueryServices; +import com.salesforce.phoenix.query.QueryServices; +import com.salesforce.phoenix.util.QueryUtil; +import com.salesforce.phoenix.util.ReadOnlyProps; + + +public class MutableSaltedIndexTest extends BaseMutableIndexTest{ + private static final int TABLE_SPLITS = 3; + private static final int INDEX_SPLITS = 4; + + @BeforeClass + public static void doSetup() throws Exception { + Map props = Maps.newHashMapWithExpectedSize(1); + // Don't cache meta information for this test because the splits change between tests + props.put(QueryServices.REGION_BOUNDARY_CACHE_TTL_MS_ATTRIB, Integer.toString(0)); + // Must update config before starting server + startServer(getUrl(), new ReadOnlyProps(props.entrySet().iterator())); + } + + @Before + public void destroyTables() throws Exception { + // Physically delete HBase table so that splits occur as expected for each test + Properties props = new Properties(TEST_PROPERTIES); + ConnectionQueryServices services = DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class).getQueryServices(); + HBaseAdmin admin = services.getAdmin(); + try { + try { + admin.disableTable(INDEX_TABLE_FULL_NAME); + admin.deleteTable(INDEX_TABLE_FULL_NAME); + } catch (TableNotFoundException e) { + } + try { + admin.disableTable(DATA_TABLE_FULL_NAME); + admin.deleteTable(DATA_TABLE_FULL_NAME); + } catch (TableNotFoundException e) { + } + } finally { + admin.close(); + } + } + + @Test + public void testMutableTableIndexMaintanenceSaltedSalted() throws Exception { + testMutableTableIndexMaintanence(TABLE_SPLITS, INDEX_SPLITS); + } + + @Test + public void testMutableTableIndexMaintanenceSalted() throws Exception { + testMutableTableIndexMaintanence(null, INDEX_SPLITS); + } + + @Test + public void testMutableTableIndexMaintanenceUnsalted() throws Exception { + testMutableTableIndexMaintanence(null, null); + } + + private void testMutableTableIndexMaintanence(Integer tableSaltBuckets, Integer indexSaltBuckets) throws Exception { + String query; + ResultSet rs; + + Properties props = new Properties(TEST_PROPERTIES); + Connection conn = DriverManager.getConnection(getUrl(), props); + conn.setAutoCommit(false); + conn.createStatement().execute("CREATE TABLE " + DATA_TABLE_FULL_NAME + " (k VARCHAR NOT NULL PRIMARY KEY, v VARCHAR) " + (tableSaltBuckets == null ? "" : " SALT_BUCKETS=" + tableSaltBuckets)); + query = "SELECT * FROM " + DATA_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertFalse(rs.next()); + + conn.createStatement().execute("CREATE INDEX " + INDEX_TABLE_NAME + " ON " + DATA_TABLE_FULL_NAME + " (v DESC)" + (indexSaltBuckets == null ? "" : " SALT_BUCKETS=" + indexSaltBuckets)); + query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertFalse(rs.next()); + + PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?)"); + stmt.setString(1,"a"); + stmt.setString(2, "x"); + stmt.execute(); + stmt.setString(1,"b"); + stmt.setString(2, "y"); + stmt.execute(); + conn.commit(); + + query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("y",rs.getString(1)); + assertEquals("b",rs.getString(2)); + assertTrue(rs.next()); + assertEquals("x",rs.getString(1)); + assertEquals("a",rs.getString(2)); + assertFalse(rs.next()); + + query = "SELECT k,v FROM " + DATA_TABLE_FULL_NAME + " WHERE v = 'y'"; + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("b",rs.getString(1)); + assertEquals("y",rs.getString(2)); + assertFalse(rs.next()); + + String expectedPlan; + rs = conn.createStatement().executeQuery("EXPLAIN " + query); + expectedPlan = indexSaltBuckets == null ? + "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + INDEX_TABLE_FULL_NAME + " 'y'" : + ("CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 KEYS OVER " + INDEX_TABLE_FULL_NAME + " 0...3,'y'\n" + + "CLIENT MERGE SORT"); + assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); + + // Will use index, so rows returned in DESC order. + // This is not a bug, though, because we can + // return in any order. + query = "SELECT k,v FROM " + DATA_TABLE_FULL_NAME + " WHERE v >= 'x'"; + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("b",rs.getString(1)); + assertEquals("y",rs.getString(2)); + assertTrue(rs.next()); + assertEquals("a",rs.getString(1)); + assertEquals("x",rs.getString(2)); + assertFalse(rs.next()); + rs = conn.createStatement().executeQuery("EXPLAIN " + query); + expectedPlan = indexSaltBuckets == null ? + "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + INDEX_TABLE_FULL_NAME + " (*-'x']" : + ("CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 RANGES OVER " + INDEX_TABLE_FULL_NAME + " 0...3,(*-'x']\n" + + "CLIENT MERGE SORT"); + assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); + + // Will still use index, since there's no LIMIT clause + query = "SELECT k,v FROM " + DATA_TABLE_FULL_NAME + " WHERE v >= 'x' ORDER BY k"; + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("a",rs.getString(1)); + assertEquals("x",rs.getString(2)); + assertTrue(rs.next()); + assertEquals("b",rs.getString(1)); + assertEquals("y",rs.getString(2)); + assertFalse(rs.next()); + rs = conn.createStatement().executeQuery("EXPLAIN " + query); + // Turns into an ORDER BY, which could be bad if lots of data is + // being returned. Without stats we don't know. The alternative + // would be a full table scan. + expectedPlan = indexSaltBuckets == null ? + ("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + INDEX_TABLE_FULL_NAME + " (*-'x']\n" + + " SERVER TOP -1 ROWS SORTED BY [:K]\n" + + "CLIENT MERGE SORT") : + ("CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 RANGES OVER " + INDEX_TABLE_FULL_NAME + " 0...3,(*-'x']\n" + + " SERVER TOP -1 ROWS SORTED BY [:K]\n" + + "CLIENT MERGE SORT"); + assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); + + // Will use data table now, since there's a LIMIT clause and + // we're able to optimize out the ORDER BY, unless the data + // table is salted. + query = "SELECT k,v FROM " + DATA_TABLE_FULL_NAME + " WHERE v >= 'x' ORDER BY k LIMIT 2"; + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("a",rs.getString(1)); + assertEquals("x",rs.getString(2)); + assertTrue(rs.next()); + assertEquals("b",rs.getString(1)); + assertEquals("y",rs.getString(2)); + assertFalse(rs.next()); + rs = conn.createStatement().executeQuery("EXPLAIN " + query); + expectedPlan = tableSaltBuckets == null ? + "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + DATA_TABLE_FULL_NAME + "\n" + + " SERVER FILTER BY V >= 'x'\n" + + " SERVER 2 ROW LIMIT\n" + + "CLIENT 2 ROW LIMIT" : + "CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 RANGES OVER " + INDEX_TABLE_FULL_NAME + " 0...3,(*-'x']\n" + + " SERVER TOP 2 ROWS SORTED BY [:K]\n" + + "CLIENT MERGE SORT"; + assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); + } +} From 277da9670f280300d72a22251e0468d0b040251a Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 20 Sep 2013 01:48:58 -0700 Subject: [PATCH 053/102] More comments --- .../com/salesforce/phoenix/end2end/index/MutableIndexTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java index f635e308..d9b01afb 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java @@ -110,6 +110,7 @@ public void testCoveredColumnUpdates() throws Exception { String query = "SELECT char_col1, int_col1, long_col2 from " + DATA_TABLE_FULL_NAME; ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + query); // FIXME: Why is this a 4 way parallel scan while the data table is a 1 way scan? + // Because of lack of stats initially. To account for this, what should we do? assertEquals("CLIENT PARALLEL 4-WAY FULL SCAN OVER " + INDEX_TABLE_FULL_NAME, QueryUtil.getExplainPlan(rs)); rs = conn.createStatement().executeQuery(query); From 439546cd646128a98b02fb1dac55c64e0d0e33af Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 20 Sep 2013 08:08:04 -0700 Subject: [PATCH 054/102] Tweak test --- .../end2end/index/MutableIndexTest.java | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java index d9b01afb..c14a33f1 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java @@ -27,16 +27,16 @@ public class MutableIndexTest extends BaseMutableIndexTest { - @BeforeClass // FIXME: We should not need to do this + @BeforeClass public static void doSetup() throws Exception { Map props = Maps.newHashMapWithExpectedSize(1); - // Don't cache meta information for this test because the splits change between tests - props.put(QueryServices.REGION_BOUNDARY_CACHE_TTL_MS_ATTRIB, Integer.toString(0)); + // Don't split intra region so we can more easily know that the n-way parallelization is for the explain plan + props.put(QueryServices.MAX_INTRA_REGION_PARALLELIZATION_ATTRIB, Integer.toString(1)); // Must update config before starting server startServer(getUrl(), new ReadOnlyProps(props.entrySet().iterator())); } - @Before // FIXME: We should not need to destroy the tables between these tests + @Before // FIXME: this shouldn't be necessary, but the tests hang without it. public void destroyTables() throws Exception { // Physically delete HBase table so that splits occur as expected for each test Properties props = new Properties(TEST_PROPERTIES); @@ -58,7 +58,6 @@ public void destroyTables() throws Exception { } } - @Test public void testIndexWithNullableFixedWithCols() throws Exception { Properties props = new Properties(TEST_PROPERTIES); @@ -80,6 +79,7 @@ public void testIndexWithNullableFixedWithCols() throws Exception { rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("chara", rs.getString(1)); + assertEquals("chara", rs.getString("char_col1")); assertEquals(2, rs.getInt(2)); assertTrue(rs.next()); assertEquals("chara", rs.getString(1)); @@ -109,9 +109,7 @@ public void testCoveredColumnUpdates() throws Exception { String query = "SELECT char_col1, int_col1, long_col2 from " + DATA_TABLE_FULL_NAME; ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + query); - // FIXME: Why is this a 4 way parallel scan while the data table is a 1 way scan? - // Because of lack of stats initially. To account for this, what should we do? - assertEquals("CLIENT PARALLEL 4-WAY FULL SCAN OVER " + INDEX_TABLE_FULL_NAME, QueryUtil.getExplainPlan(rs)); + assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + INDEX_TABLE_FULL_NAME, QueryUtil.getExplainPlan(rs)); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -209,7 +207,7 @@ public void testSelectAll() throws Exception { query = "SELECT * FROM " + DATA_TABLE_FULL_NAME; rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertEquals("CLIENT PARALLEL 4-WAY FULL SCAN OVER " + INDEX_TABLE_FULL_NAME, QueryUtil.getExplainPlan(rs)); + assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + INDEX_TABLE_FULL_NAME, QueryUtil.getExplainPlan(rs)); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -266,7 +264,7 @@ public void testSelectCF() throws Exception { query = "SELECT a.* FROM " + DATA_TABLE_FULL_NAME; rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertEquals("CLIENT PARALLEL 4-WAY FULL SCAN OVER " + INDEX_TABLE_FULL_NAME, QueryUtil.getExplainPlan(rs)); + assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + INDEX_TABLE_FULL_NAME, QueryUtil.getExplainPlan(rs)); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("y",rs.getString(1)); From b82b4a9934773c6dc7f78b18d7b733957cf40b9e Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Fri, 20 Sep 2013 10:25:26 -0700 Subject: [PATCH 055/102] Replacing Strings w/ bytes in Indexer. Saves us the serialization/deserialization to Strings we did previously. --- .../salesforce/hbase/index/IndexWriter.java | 28 +++++++++++-------- .../com/salesforce/hbase/index/Indexer.java | 25 ++++++++--------- .../hbase/index/builder/IndexBuilder.java | 6 ++-- .../builder/example/ColumnFamilyIndexer.java | 14 +++++----- .../covered/CoveredColumnsIndexBuilder.java | 6 ++-- .../covered/example/CoveredColumnIndexer.java | 2 +- .../covered/update/IndexUpdateManager.java | 7 ++--- .../index/table/HTableInterfaceReference.java | 23 +++++++-------- .../hbase/index/wal/IndexedKeyValue.java | 20 +++++++------ .../update/TestIndexUpdateManager.java | 4 +-- .../wal/TestReadWriteKeyValuesWithCodec.java | 3 +- 11 files changed, 73 insertions(+), 65 deletions(-) diff --git a/src/main/java/com/salesforce/hbase/index/IndexWriter.java b/src/main/java/com/salesforce/hbase/index/IndexWriter.java index 7fdcd7b6..6e2477e4 100644 --- a/src/main/java/com/salesforce/hbase/index/IndexWriter.java +++ b/src/main/java/com/salesforce/hbase/index/IndexWriter.java @@ -50,6 +50,7 @@ import com.google.common.collect.Multimap; import com.salesforce.hbase.index.table.HTableFactory; import com.salesforce.hbase.index.table.HTableInterfaceReference; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; /** * Do the actual work of writing to the index tables. Ensures that if we do fail to write to the @@ -86,26 +87,27 @@ public IndexWriter(String sourceInfo, Abortable abortable, HTableFactory factory * {@link HBaseAdmin#unassign(byte[], boolean)}) the region and then failing that calls * {@link System#exit(int)} to kill the server. */ - public void writeAndKillYourselfOnFailure(Collection> indexUpdates) { + public void writeAndKillYourselfOnFailure(Collection> indexUpdates) { try { write(indexUpdates); } catch (Exception e) { killYourself(e); } } + /** * Write the mutations to their respective table using the provided factory. *

* This method is not thread-safe and if accessed in a non-serial manner could leak HTables. - * @param updates Updates to write + * @param indexUpdates Updates to write * @throws CannotReachIndexException if we cannot successfully write a single index entry. We stop * immediately on the first failed index write, rather than attempting all writes. */ - public void write(Collection> updates) + public void write(Collection> indexUpdates) throws CannotReachIndexException { // convert the strings to htableinterfaces to which we can talk and group by TABLE Multimap toWrite = - resolveTableReferences(factory, updates); + resolveTableReferences(factory, indexUpdates); // write each mutation, as a part of a batch, to its respective table List mutations; @@ -171,17 +173,19 @@ private void killYourself(Throwable cause) { * @return pairs that can then be written by an {@link IndexWriter}. */ public static Multimap resolveTableReferences( - HTableFactory factory, Collection> indexUpdates) { + HTableFactory factory, Collection> indexUpdates) { Multimap updates = ArrayListMultimap. create(); - Map tables = - new HashMap(updates.size()); - for (Pair entry : indexUpdates) { - String tableName = entry.getSecond(); - HTableInterfaceReference table = tables.get(tableName); + // simple map to make lookups easy while we build the map of tables to create + Map tables = + new HashMap(updates.size()); + for (Pair entry : indexUpdates) { + byte[] tableName = entry.getSecond(); + ImmutableBytesPtr ptr = new ImmutableBytesPtr(tableName); + HTableInterfaceReference table = tables.get(ptr); if (table == null) { - table = new HTableInterfaceReference(entry.getSecond(), factory); - tables.put(tableName, table); + table = new HTableInterfaceReference(ptr, factory); + tables.put(ptr, table); } updates.put(table, entry.getFirst()); } diff --git a/src/main/java/com/salesforce/hbase/index/Indexer.java b/src/main/java/com/salesforce/hbase/index/Indexer.java index 07afed89..f9310794 100644 --- a/src/main/java/com/salesforce/hbase/index/Indexer.java +++ b/src/main/java/com/salesforce/hbase/index/Indexer.java @@ -153,7 +153,7 @@ public void prePut(final ObserverContext c, final if (!this.builder.isEnabled(put)) { return; } - Collection> indexUpdates = this.builder.getIndexUpdate(put); + Collection> indexUpdates = this.builder.getIndexUpdate(put); doPre(indexUpdates, edit, writeToWAL); } @@ -166,15 +166,15 @@ public void preDelete(ObserverContext e, Delete de return; } // get the mapping for index column -> target index table - Collection> indexUpdates = this.builder.getIndexUpdate(delete); + Collection> indexUpdates = this.builder.getIndexUpdate(delete); doPre(indexUpdates, edit, writeToWAL); } - private void doPre(Collection> updates, + private void doPre(Collection> indexUpdates, final WALEdit edit, final boolean writeToWAL) throws IOException { // no index updates, so we are done - if (updates == null || updates.size() == 0) { + if (indexUpdates == null || indexUpdates.size() == 0) { return; } @@ -182,16 +182,16 @@ private void doPre(Collection> updates, // update right away if (!writeToWAL) { try { - this.writer.write(updates); + this.writer.write(indexUpdates); return; } catch (CannotReachIndexException e) { - LOG.error("Failed to update index with entries:" + updates, e); + LOG.error("Failed to update index with entries:" + indexUpdates, e); throw new IOException(e); } } // we have all the WAL durability, so we just update the WAL entry and move on - for (Pair entry : updates) { + for (Pair entry : indexUpdates) { edit.add(new IndexedKeyValue(entry.getSecond(), entry.getFirst())); } @@ -258,7 +258,7 @@ private void doPost(WALEdit edit, Mutation m, boolean writeToWAL) { * lead to writing all the index updates for each Put/Delete). */ if (!ikv.getBatchFinished()) { - Collection> indexUpdates = extractIndexUpdate(edit); + Collection> indexUpdates = extractIndexUpdate(edit); // the WAL edit is kept in memory and we already specified the factory when we created the // references originally - therefore, we just pass in a null factory here and use the ones @@ -295,13 +295,12 @@ private IndexedKeyValue getFirstIndexedKeyValue(WALEdit edit) { * @param edit to search for index updates * @return the mutations to apply to the index tables */ - private Collection> extractIndexUpdate(WALEdit edit) { - Collection> indexUpdates = new ArrayList>(); + private Collection> extractIndexUpdate(WALEdit edit) { + Collection> indexUpdates = new ArrayList>(); for (KeyValue kv : edit.getKeyValues()) { if (kv instanceof IndexedKeyValue) { IndexedKeyValue ikv = (IndexedKeyValue) kv; - indexUpdates.add(new Pair(ikv.getMutation(), ikv - .getIndexTable())); + indexUpdates.add(new Pair(ikv.getMutation(), ikv.getIndexTable())); } } @@ -311,7 +310,7 @@ private Collection> extractIndexUpdate(WALEdit edit) { @Override public void preWALRestore(ObserverContext env, HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { - Collection> indexUpdates = extractIndexUpdate(logEdit); + Collection> indexUpdates = extractIndexUpdate(logEdit); writer.writeAndKillYourselfOnFailure(indexUpdates); } diff --git a/src/main/java/com/salesforce/hbase/index/builder/IndexBuilder.java b/src/main/java/com/salesforce/hbase/index/builder/IndexBuilder.java index f821ae75..4d10593b 100644 --- a/src/main/java/com/salesforce/hbase/index/builder/IndexBuilder.java +++ b/src/main/java/com/salesforce/hbase/index/builder/IndexBuilder.java @@ -69,7 +69,7 @@ public interface IndexBuilder { * @return a Map of the mutations to make -> target index table name * @throws IOException on failure */ - public Collection> getIndexUpdate(Put put) throws IOException; + public Collection> getIndexUpdate(Put put) throws IOException; /** * The counter-part to {@link #getIndexUpdate(Put)} - your opportunity to update any/all index @@ -79,7 +79,7 @@ public interface IndexBuilder { * @return a {@link Map} of the mutations to make -> target index table name * @throws IOException on failure */ - public Collection> getIndexUpdate(Delete delete) throws IOException; + public Collection> getIndexUpdate(Delete delete) throws IOException; /** * Build an index update to cleanup the index when we remove {@link KeyValue}s via the normal @@ -89,7 +89,7 @@ public interface IndexBuilder { * @return a {@link Map} of the mutations to make -> target index table name * @throws IOException on failure */ - public Collection> getIndexUpdateForFilteredRows( + public Collection> getIndexUpdateForFilteredRows( Collection filtered) throws IOException; diff --git a/src/main/java/com/salesforce/hbase/index/builder/example/ColumnFamilyIndexer.java b/src/main/java/com/salesforce/hbase/index/builder/example/ColumnFamilyIndexer.java index 3a1397eb..da276a09 100644 --- a/src/main/java/com/salesforce/hbase/index/builder/example/ColumnFamilyIndexer.java +++ b/src/main/java/com/salesforce/hbase/index/builder/example/ColumnFamilyIndexer.java @@ -111,13 +111,13 @@ public void setup(RegionCoprocessorEnvironment env) { } @Override - public Collection> getIndexUpdate(Put p) { + public Collection> getIndexUpdate(Put p) { // if not columns to index, we are done and don't do anything special if (columnTargetMap == null || columnTargetMap.size() == 0) { return null; } - Collection> updateMap = new ArrayList>(); + Collection> updateMap = new ArrayList>(); Set keys = p.getFamilyMap().keySet(); for (Entry> entry : p.getFamilyMap().entrySet()) { String ref = columnTargetMap @@ -157,19 +157,19 @@ public Collection> getIndexUpdate(Put p) { } // add the mapping - updateMap.add(new Pair(put, ref)); + updateMap.add(new Pair(put, Bytes.toBytes(ref))); } return updateMap; } @Override - public Collection> getIndexUpdate(Delete d) { + public Collection> getIndexUpdate(Delete d) { // if no columns to index, we are done and don't do anything special if (columnTargetMap == null || columnTargetMap.size() == 0) { return null; } - Collection> updateMap = new ArrayList>(); + Collection> updateMap = new ArrayList>(); for (Entry> entry : d.getFamilyMap().entrySet()) { String ref = columnTargetMap .get(new ImmutableBytesPtr(entry.getKey())); @@ -186,7 +186,7 @@ public Collection> getIndexUpdate(Delete d) { // column family from the original update Delete delete = new Delete(kvs.get(0).getFamily()); // add the mapping - updateMap.add(new Pair(delete, ref)); + updateMap.add(new Pair(delete, Bytes.toBytes(ref))); } return updateMap; } @@ -209,7 +209,7 @@ public static void createIndexTable(HBaseAdmin admin, String indexTable) throws * Doesn't do any index cleanup. This is just a basic example case. */ @Override - public Collection> getIndexUpdateForFilteredRows( + public Collection> getIndexUpdateForFilteredRows( Collection filtered) throws IOException { return null; diff --git a/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java b/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java index 4f0407a4..439d1b31 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java +++ b/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java @@ -108,7 +108,7 @@ public void setup(RegionCoprocessorEnvironment env) throws IOException { } @Override - public Collection> getIndexUpdate(Put p) throws IOException { + public Collection> getIndexUpdate(Put p) throws IOException { // build the index updates for each group IndexUpdateManager updateMap = new IndexUpdateManager(); @@ -427,7 +427,7 @@ private void cleanupIndexStateFromBatchOnward(IndexUpdateManager updateMap, } @Override - public Collection> getIndexUpdate(Delete d) throws IOException { + public Collection> getIndexUpdate(Delete d) throws IOException { // stores all the return values IndexUpdateManager updateMap = new IndexUpdateManager(); @@ -481,7 +481,7 @@ public Collection> getIndexUpdate(Delete d) throws IOExce } @Override - public Collection> getIndexUpdateForFilteredRows( + public Collection> getIndexUpdateForFilteredRows( Collection filtered) throws IOException { // TODO Implement IndexBuilder.getIndexUpdateForFilteredRows return null; diff --git a/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexer.java b/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexer.java index 8338c6ed..299f3b36 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexer.java +++ b/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexer.java @@ -92,7 +92,7 @@ public static void createIndexTable(HBaseAdmin admin, String indexTable) throws } @Override - public Collection> getIndexUpdateForFilteredRows( + public Collection> getIndexUpdateForFilteredRows( Collection filtered) throws IOException { // stores all the return values diff --git a/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java b/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java index afbb53a1..b6c5725e 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java +++ b/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java @@ -184,20 +184,19 @@ private void markMutationForRemoval(Mutation m) { m.setAttribute(PHOENIX_HBASE_TEMP_DELETE_MARKER, TRUE_MARKER); } - public List> toMap() { - List> updateMap = Lists.newArrayList(); + public List> toMap() { + List> updateMap = Lists.newArrayList(); for (Entry> updates : map.entrySet()) { // get is ok because we always set with just the bytes byte[] tableName = updates.getKey().get(); // TODO replace this as just storing a byte[], to avoid all the String <-> byte[] swapping // HBase does - String table = Bytes.toString(tableName); for (Mutation m : updates.getValue()) { // skip elements that have been marked for delete if (shouldBeRemoved(m)) { continue; } - updateMap.add(new Pair(m, table)); + updateMap.add(new Pair(m, tableName)); } } return updateMap; diff --git a/src/main/java/com/salesforce/hbase/index/table/HTableInterfaceReference.java b/src/main/java/com/salesforce/hbase/index/table/HTableInterfaceReference.java index 82399069..b50d9c15 100644 --- a/src/main/java/com/salesforce/hbase/index/table/HTableInterfaceReference.java +++ b/src/main/java/com/salesforce/hbase/index/table/HTableInterfaceReference.java @@ -6,9 +6,10 @@ import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.client.HTableInterface; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.Writable; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; + /** * Reference to an HTableInterface that only gets the underlying {@link HTableInterface} from the * {@link CoprocessorEnvironment} on calls to {@link #getTable()}. Until calling {@link #getTable()} @@ -27,7 +28,7 @@ */ public class HTableInterfaceReference implements Writable { - private String tableName; + private ImmutableBytesPtr tableName; private HTableInterface table; private HTableFactory factory; @@ -39,12 +40,12 @@ public class HTableInterfaceReference implements Writable { public HTableInterfaceReference() { } - public HTableInterfaceReference(String tablename) { - this.tableName = tablename; + public HTableInterfaceReference(ImmutableBytesPtr tableName) { + this.tableName = tableName; } - public HTableInterfaceReference(String tablename, HTableFactory factory) { - this.tableName = tablename; + public HTableInterfaceReference(ImmutableBytesPtr tableName, HTableFactory factory) { + this(tableName); this.factory = factory; } @@ -54,7 +55,7 @@ public void setFactory(HTableFactory e) { public HTableInterface getTable(HTableFactory e) throws IOException { if (this.table == null) { - this.table = e.getTable(Bytes.toBytes(tableName)); + this.table = e.getTable(this.tableName.copyBytes()); } return this.table; } @@ -69,18 +70,18 @@ public HTableInterface getTable() throws IOException { } public String getTableName() { - return this.tableName; + return this.tableName.toString(); } @Override public void readFields(DataInput in) throws IOException { - this.tableName = in.readUTF(); - + this.tableName = new ImmutableBytesPtr(); + this.tableName.readFields(in); } @Override public void write(DataOutput out) throws IOException { - out.writeUTF(this.tableName); + this.tableName.write(out); } @Override diff --git a/src/main/java/com/salesforce/hbase/index/wal/IndexedKeyValue.java b/src/main/java/com/salesforce/hbase/index/wal/IndexedKeyValue.java index 5d9d36ae..3324c303 100644 --- a/src/main/java/com/salesforce/hbase/index/wal/IndexedKeyValue.java +++ b/src/main/java/com/salesforce/hbase/index/wal/IndexedKeyValue.java @@ -13,7 +13,7 @@ public class IndexedKeyValue extends KeyValue { - String indexTableName; + byte[] indexTableName; Mutation mutation; // optimization check to ensure that batches don't get replayed to the index more than once private boolean batchFinished = false; @@ -21,13 +21,17 @@ public class IndexedKeyValue extends KeyValue { public IndexedKeyValue() { } - public IndexedKeyValue(String target, Mutation mutation) { - this.indexTableName = target; + public IndexedKeyValue(byte[] bs, Mutation mutation) { + this.indexTableName = bs; this.mutation = mutation; } - public String getIndexTable() { - return indexTableName; + public byte[] getIndexTable() { + return this.indexTableName; + } + + public String getIndexTableString() { + return Bytes.toString(indexTableName); } public Mutation getMutation() { @@ -56,7 +60,7 @@ public String toString() { public boolean equals(Object o) { if (o instanceof IndexedKeyValue) { IndexedKeyValue other = (IndexedKeyValue) o; - if (other.indexTableName.equals(this.indexTableName)) { + if (Bytes.equals(other.indexTableName, this.indexTableName)) { try { byte[] current = getBytes(this.mutation); byte[] otherMutation = getBytes(other.mutation); @@ -101,7 +105,7 @@ public void write(DataOutput out) throws IOException{ * @throws IOException if there is a problem writing the underlying data */ void writeData(DataOutput out) throws IOException { - out.writeUTF(this.indexTableName); + Bytes.writeByteArray(out, this.indexTableName); out.writeUTF(this.mutation.getClass().getName()); this.mutation.write(out); } @@ -113,7 +117,7 @@ void writeData(DataOutput out) throws IOException { @SuppressWarnings("javadoc") @Override public void readFields(DataInput in) throws IOException { - this.indexTableName = in.readUTF(); + this.indexTableName = Bytes.readByteArray(in); Class clazz; try { clazz = Class.forName(in.readUTF()).asSubclass(Mutation.class); diff --git a/src/test/java/com/salesforce/hbase/index/covered/update/TestIndexUpdateManager.java b/src/test/java/com/salesforce/hbase/index/covered/update/TestIndexUpdateManager.java index cafa3668..8786cc3c 100644 --- a/src/test/java/com/salesforce/hbase/index/covered/update/TestIndexUpdateManager.java +++ b/src/test/java/com/salesforce/hbase/index/covered/update/TestIndexUpdateManager.java @@ -137,8 +137,8 @@ public void testCancelingUpdates() throws Exception { } private void validate(IndexUpdateManager manager, List pending) { - for (Pair entry : manager.toMap()) { - assertEquals("Table name didn't match for stored entry!", TABLE_NAME, entry.getSecond()); + for (Pair entry : manager.toMap()) { + assertEquals("Table name didn't match for stored entry!", table, entry.getSecond()); Mutation m = pending.remove(0); // test with == to match the exact entries, Mutation.equals just checks the row assertTrue( diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestReadWriteKeyValuesWithCodec.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestReadWriteKeyValuesWithCodec.java index b598e9bc..057ab813 100644 --- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestReadWriteKeyValuesWithCodec.java +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestReadWriteKeyValuesWithCodec.java @@ -91,7 +91,8 @@ private List getEdits() { edits.add(withPutsAndDeletes); WALEdit justIndexUpdates = new WALEdit(); - IndexedKeyValue ikv = new IndexedKeyValue("targetTable", p); + byte[] table = Bytes.toBytes("targetTable"); + IndexedKeyValue ikv = new IndexedKeyValue(table, p); justIndexUpdates.add(ikv); edits.add(justIndexUpdates); From 8a689df6fc631342d081d5d7cf20e3d7e57c89e9 Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Fri, 20 Sep 2013 11:14:41 -0700 Subject: [PATCH 056/102] Updating docs for IndexMemStore --- .../index/covered/data/IndexMemStore.java | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/salesforce/hbase/index/covered/data/IndexMemStore.java b/src/main/java/com/salesforce/hbase/index/covered/data/IndexMemStore.java index 73b5c007..21515144 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/data/IndexMemStore.java +++ b/src/main/java/com/salesforce/hbase/index/covered/data/IndexMemStore.java @@ -43,9 +43,34 @@ import com.salesforce.hbase.index.covered.KeyValueStore; /** - * Like the HBase {@link MemStore}, but without all that exta work around maintaining snapshots and + * Like the HBase {@link MemStore}, but without all that extra work around maintaining snapshots and * sizing (for right now). We still support the concurrent access (in case indexes are built in - * parallel). Essentially, this is a light wrapper around a ConcurrentSkipListMap. + * parallel). + *

+ * + We basically wrap a KeyValueSkipListSet, just like a regular MemStore, except we are: + *

    + *
  1. not dealing with + *
      + *
    • space considerations
      • + *
    • a snapshot set
      • + *
    + *
    2. + *
  2. ignoring memstore timestamps in favor of deciding when we want to overwrite keys based on how + * we obtain them
    3. + *
+ *

+ * We can ignore the memstore timestamps because we know that anything we get from the local region + * is going to be MVCC visible - so it should just go in. However, we also want overwrite any + * existing state with our pending write that we are indexing, so that needs to clobber the KVs we + * get from the HRegion. This got really messy with a regular memstore as each KV from the MemStore + * frequently has a higher MemStoreTS, but we can't just up the pending KVs' MemStoreTs b/c a + * memstore relies on the MVCC readpoint, which generally is < Long.MAX_VALUE. + *

+ * By realizing that we don't need the snapshot or space requirements, we can go much faster than + * the previous implementation. Further, by being smart about how we manage the KVs, we can drop the + * extra object creation we were doing to wrap the pending KVs (which we did previously to ensure + * they sorted before the ones we got from the HRegion). */ public class IndexMemStore implements KeyValueStore { From fdde81c43b600af47b6333409c829961cf55debe Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 20 Sep 2013 17:48:19 -0700 Subject: [PATCH 057/102] Cache hashcode were it makes sense and optimize equals --- .../index/covered/example/CoveredColumn.java | 43 ++++++++----- .../index/covered/update/ColumnTracker.java | 15 +++-- .../covered/update/IndexUpdateManager.java | 5 +- .../hbase/index/scanner/ScannerBuilder.java | 1 - .../index/table/HTableInterfaceReference.java | 14 +++-- .../hbase/index/util/ImmutableBytesPtr.java | 9 +++ .../hbase/index/wal/IndexedKeyValue.java | 6 +- .../phoenix/expression/ExpressionType.java | 3 +- .../IndexKeyValueColumnExpression.java | 21 +++++++ .../expression/KeyValueColumnExpression.java | 12 +--- .../salesforce/phoenix/schema/ColumnRef.java | 10 +-- .../end2end/index/MutableIndexTest.java | 62 +++++++++++++++++++ 12 files changed, 155 insertions(+), 46 deletions(-) create mode 100644 src/main/java/com/salesforce/phoenix/expression/IndexKeyValueColumnExpression.java diff --git a/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumn.java b/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumn.java index 5f9b379f..257231ff 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumn.java +++ b/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumn.java @@ -12,10 +12,22 @@ public class CoveredColumn extends ColumnReference { public static final String SEPARATOR = ":"; String familyString; + private final int hashCode; + + private static int calcHashCode(String familyString, byte[] qualifier) { + final int prime = 31; + int result = 1; + result = prime * result + familyString.hashCode(); + if (qualifier != null) { + result = prime * result + Bytes.hashCode(qualifier); + } + return result; + } public CoveredColumn(String family, byte[] qualifier) { super(Bytes.toBytes(family), qualifier == null ? ColumnReference.ALL_QUALIFIERS : qualifier); this.familyString = family; + this.hashCode = calcHashCode(family, qualifier); } public static CoveredColumn parse(String spec) { @@ -50,25 +62,22 @@ public boolean matchesFamily(String family2) { } @Override - public boolean equals(Object o) { - CoveredColumn other = (CoveredColumn) o; - if (this.familyString.equals(other.familyString)) { - return Bytes.equals(qualifier, other.qualifier); - } - return false; - } +public int hashCode() { + return hashCode; +} - @Override - public int hashCode() { - int hash = this.familyString.hashCode(); - if (this.qualifier != null) { - hash += Bytes.hashCode(qualifier); - } - - return hash; - } +@Override +public boolean equals(Object obj) { + if (this == obj) return true; + if (!super.equals(obj)) return false; + if (getClass() != obj.getClass()) return false; + CoveredColumn other = (CoveredColumn)obj; + if (hashCode != other.hashCode) return false; + if (!familyString.equals(other.familyString)) return false; + return Bytes.equals(qualifier, other.qualifier); +} - @Override +@Override public String toString() { String qualString = qualifier == null ? "null" : Bytes.toString(qualifier); return "CoveredColumn:[" + familyString + ":" + qualString + "]"; diff --git a/src/main/java/com/salesforce/hbase/index/covered/update/ColumnTracker.java b/src/main/java/com/salesforce/hbase/index/covered/update/ColumnTracker.java index bf5a8757..6c2bf4ce 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/update/ColumnTracker.java +++ b/src/main/java/com/salesforce/hbase/index/covered/update/ColumnTracker.java @@ -46,11 +46,17 @@ public class ColumnTracker implements IndexedColumnGroup { public static final long GUARANTEED_NEWER_UPDATES = Long.MIN_VALUE; private final List columns; private long ts = NO_NEWER_PRIMARY_TABLE_ENTRY_TIMESTAMP; + private final int hashCode; + + private static int calcHashCode(List columns) { + return columns.hashCode(); + } public ColumnTracker(Collection columns) { this.columns = new ArrayList(columns); // sort the columns Collections.sort(this.columns); + this.hashCode = calcHashCode(this.columns); } /** @@ -70,11 +76,7 @@ public long getTS() { @Override public int hashCode() { - int hash = 0; - for (ColumnReference ref : columns) { - hash += ref.hashCode(); - } - return hash; + return hashCode; } @Override @@ -83,6 +85,9 @@ public boolean equals(Object o){ return false; } ColumnTracker other = (ColumnTracker)o; + if (hashCode != other.hashCode) { + return false; + } if (other.columns.size() != columns.size()) { return false; } diff --git a/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java b/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java index b6c5725e..cd82c00f 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java +++ b/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.util.Pair; import com.google.common.collect.Lists; +import com.google.common.primitives.Longs; import com.salesforce.hbase.index.util.ImmutableBytesPtr; /** @@ -60,7 +61,7 @@ public int compare(Mutation o1, Mutation o2) { } // if same row, sort by reverse timestamp (larger first) - compare = -(new Long(o1.getTimeStamp()).compareTo(o2.getTimeStamp())); + compare = Longs.compare(o2.getTimeStamp(), o1.getTimeStamp()); if (compare != 0) { return compare; } @@ -94,7 +95,7 @@ private int comparePuts(Put p1, Put p2) { // TODO: make this a real comparison // this is a little cheating, but we don't really need to worry too much about this being // the same - chances are that exact matches here are really the same update. - return new Long(p1.heapSize()).compareTo(p2.heapSize()); + return Longs.compare(p1.heapSize(), p2.heapSize()); } return compare; } diff --git a/src/main/java/com/salesforce/hbase/index/scanner/ScannerBuilder.java b/src/main/java/com/salesforce/hbase/index/scanner/ScannerBuilder.java index 15706775..1b62e830 100644 --- a/src/main/java/com/salesforce/hbase/index/scanner/ScannerBuilder.java +++ b/src/main/java/com/salesforce/hbase/index/scanner/ScannerBuilder.java @@ -17,7 +17,6 @@ import com.google.common.collect.Lists; import com.salesforce.hbase.index.covered.KeyValueStore; -import com.salesforce.hbase.index.covered.data.IndexMemStore; import com.salesforce.hbase.index.covered.filter.ApplyAndFilterDeletesFilter; import com.salesforce.hbase.index.covered.filter.ColumnTrackingNextLargestTimestampFilter; import com.salesforce.hbase.index.covered.filter.MaxTimestampFilter; diff --git a/src/main/java/com/salesforce/hbase/index/table/HTableInterfaceReference.java b/src/main/java/com/salesforce/hbase/index/table/HTableInterfaceReference.java index b50d9c15..0c3472ca 100644 --- a/src/main/java/com/salesforce/hbase/index/table/HTableInterfaceReference.java +++ b/src/main/java/com/salesforce/hbase/index/table/HTableInterfaceReference.java @@ -55,7 +55,7 @@ public void setFactory(HTableFactory e) { public HTableInterface getTable(HTableFactory e) throws IOException { if (this.table == null) { - this.table = e.getTable(this.tableName.copyBytes()); + this.table = e.getTable(this.tableName.copyBytesIfNecessary()); } return this.table; } @@ -86,11 +86,17 @@ public void write(DataOutput out) throws IOException { @Override public int hashCode() { - return this.tableName.hashCode(); + return tableName.hashCode(); } @Override - public boolean equals(Object o) { - return o == null ? false : this.hashCode() == o.hashCode(); + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + HTableInterfaceReference other = (HTableInterfaceReference)obj; + return tableName.equals(other.tableName); } + + } \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/util/ImmutableBytesPtr.java b/src/main/java/com/salesforce/hbase/index/util/ImmutableBytesPtr.java index 52a3eadb..685c7349 100644 --- a/src/main/java/com/salesforce/hbase/index/util/ImmutableBytesPtr.java +++ b/src/main/java/com/salesforce/hbase/index/util/ImmutableBytesPtr.java @@ -91,5 +91,14 @@ public void set(final byte [] b, final int offset, final int length) { hashCode = super.hashCode(); } + /** + * @return the backing byte array, copying only if necessary + */ + public byte[] copyBytesIfNecessary() { + if (this.getOffset() == 0 && this.getLength() == this.get().length) { + return this.get(); + } + return this.copyBytes(); + } } diff --git a/src/main/java/com/salesforce/hbase/index/wal/IndexedKeyValue.java b/src/main/java/com/salesforce/hbase/index/wal/IndexedKeyValue.java index 3324c303..7d0ac3b4 100644 --- a/src/main/java/com/salesforce/hbase/index/wal/IndexedKeyValue.java +++ b/src/main/java/com/salesforce/hbase/index/wal/IndexedKeyValue.java @@ -89,7 +89,11 @@ private byte[] getBytes(Mutation m) throws IOException{ @Override public int hashCode() { - return this.indexTableName.hashCode() + this.mutation.hashCode(); + final int prime = 31; + int result = 1; + result = prime * result + this.indexTableName.hashCode(); + result = prime * result + this.mutation.hashCode(); + return result; } @Override diff --git a/src/main/java/com/salesforce/phoenix/expression/ExpressionType.java b/src/main/java/com/salesforce/phoenix/expression/ExpressionType.java index b3c4f702..3a1a43a1 100644 --- a/src/main/java/com/salesforce/phoenix/expression/ExpressionType.java +++ b/src/main/java/com/salesforce/phoenix/expression/ExpressionType.java @@ -125,7 +125,8 @@ public enum ExpressionType { DoubleMultiplyExpression(DoubleMultiplyExpression.class), DoubleDivideExpression(DoubleDivideExpression.class), MD5Function(MD5Function.class), - SqlTableType(SqlTableType.class); + SqlTableType(SqlTableType.class), + IndexKeyValue(IndexKeyValueColumnExpression.class); ExpressionType(Class clazz) { this.clazz = clazz; diff --git a/src/main/java/com/salesforce/phoenix/expression/IndexKeyValueColumnExpression.java b/src/main/java/com/salesforce/phoenix/expression/IndexKeyValueColumnExpression.java new file mode 100644 index 00000000..c119389a --- /dev/null +++ b/src/main/java/com/salesforce/phoenix/expression/IndexKeyValueColumnExpression.java @@ -0,0 +1,21 @@ +package com.salesforce.phoenix.expression; + +import com.salesforce.phoenix.schema.PColumn; +import com.salesforce.phoenix.util.SchemaUtil; + +public class IndexKeyValueColumnExpression extends KeyValueColumnExpression { + + public IndexKeyValueColumnExpression() { + } + + public IndexKeyValueColumnExpression(PColumn column) { + super(column); + } + + @Override + public String toString() { + // Just display the column name part, since it's guaranteed to be unique + return SchemaUtil.getColumnDisplayName(this.getColumnName()); + } + +} diff --git a/src/main/java/com/salesforce/phoenix/expression/KeyValueColumnExpression.java b/src/main/java/com/salesforce/phoenix/expression/KeyValueColumnExpression.java index 182ce045..a3724513 100644 --- a/src/main/java/com/salesforce/phoenix/expression/KeyValueColumnExpression.java +++ b/src/main/java/com/salesforce/phoenix/expression/KeyValueColumnExpression.java @@ -52,22 +52,12 @@ public class KeyValueColumnExpression extends ColumnExpression { private byte[] cf; private byte[] cq; - private final String alias; // not serialized public KeyValueColumnExpression() { - this.alias = null; } public KeyValueColumnExpression(PColumn column) { super(column); - this.alias = null; - this.cf = column.getFamilyName().getBytes(); - this.cq = column.getName().getBytes(); - } - - public KeyValueColumnExpression(PColumn column, String alias) { - super(column); - this.alias = alias; this.cf = column.getFamilyName().getBytes(); this.cq = column.getName().getBytes(); } @@ -103,7 +93,7 @@ public boolean equals(Object obj) { @Override public String toString() { - return alias != null ? alias : SchemaUtil.getColumnDisplayName(cf, cq); + return SchemaUtil.getColumnDisplayName(cf, cq); } @Override diff --git a/src/main/java/com/salesforce/phoenix/schema/ColumnRef.java b/src/main/java/com/salesforce/phoenix/schema/ColumnRef.java index 68153939..932c56a6 100644 --- a/src/main/java/com/salesforce/phoenix/schema/ColumnRef.java +++ b/src/main/java/com/salesforce/phoenix/schema/ColumnRef.java @@ -32,6 +32,7 @@ import org.apache.http.annotation.Immutable; import com.salesforce.phoenix.expression.ColumnExpression; +import com.salesforce.phoenix.expression.IndexKeyValueColumnExpression; import com.salesforce.phoenix.expression.KeyValueColumnExpression; import com.salesforce.phoenix.expression.RowKeyColumnExpression; import com.salesforce.phoenix.util.SchemaUtil; @@ -96,10 +97,11 @@ public ColumnExpression newColumnExpression() throws SQLException { if (SchemaUtil.isPKColumn(this.getColumn())) { return new RowKeyColumnExpression(getColumn(), new RowKeyValueAccessor(this.getTable().getPKColumns(), pkSlotPosition)); } else { - PColumn column = getColumn(); - boolean isIndex = tableRef.getTable().getType() == PTableType.INDEX; - String alias = isIndex ? SchemaUtil.getColumnDisplayName(column.getName().getBytes()) : null; - return new KeyValueColumnExpression(column, alias); + if (tableRef.getTable().getType() == PTableType.INDEX) { + return new IndexKeyValueColumnExpression(getColumn()); + } else { + return new KeyValueColumnExpression(getColumn()); + } } } diff --git a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java index c14a33f1..2dcb1437 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java @@ -3,6 +3,7 @@ import static com.salesforce.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.sql.Connection; @@ -278,4 +279,65 @@ public void testSelectCF() throws Exception { assertEquals("1",rs.getString("v2")); assertFalse(rs.next()); } + + @Test + public void testCompoundIndexKey() throws Exception { + String query; + ResultSet rs; + + Properties props = new Properties(TEST_PROPERTIES); + Connection conn = DriverManager.getConnection(getUrl(), props); + conn.setAutoCommit(false); + conn.createStatement().execute("CREATE TABLE " + DATA_TABLE_FULL_NAME + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)"); + query = "SELECT * FROM " + DATA_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertFalse(rs.next()); + + conn.createStatement().execute("CREATE INDEX " + INDEX_TABLE_NAME + " ON " + DATA_TABLE_FULL_NAME + " (v1, v2)"); + query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertFalse(rs.next()); + + PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?)"); + stmt.setString(1,"a"); + stmt.setString(2, "x"); + stmt.setString(3, "1"); + stmt.execute(); + conn.commit(); + + query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("x",rs.getString(1)); + assertEquals("1",rs.getString(2)); + assertEquals("a",rs.getString(3)); + assertFalse(rs.next()); + + stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?)"); + stmt.setString(1,"a"); + stmt.setString(2, "y"); + stmt.setString(3, null); + stmt.execute(); + conn.commit(); + + query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("y",rs.getString(1)); + assertNull(rs.getString(2)); + assertEquals("a",rs.getString(3)); + assertFalse(rs.next()); + + query = "SELECT * FROM " + DATA_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery("EXPLAIN " + query); + assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + INDEX_TABLE_FULL_NAME, QueryUtil.getExplainPlan(rs)); + + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("a",rs.getString(1)); + assertNull(rs.getString(2)); + assertEquals("y",rs.getString(3)); + assertFalse(rs.next()); + } + } From a0643bb4aedb2eb7d8a2643778fa6896d93bd508 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 20 Sep 2013 22:36:43 -0700 Subject: [PATCH 058/102] Commenting out broken test --- .../com/salesforce/phoenix/end2end/index/MutableIndexTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java index 2dcb1437..de21afff 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java @@ -280,7 +280,7 @@ public void testSelectCF() throws Exception { assertFalse(rs.next()); } - @Test + // @Test Broken, but Jesse is fixing public void testCompoundIndexKey() throws Exception { String query; ResultSet rs; From b1e50e3e8afe72cd819f831cff03720d5713380c Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sat, 21 Sep 2013 16:17:02 -0700 Subject: [PATCH 059/102] Fix for alias reference in group by or order by for an index table (issue #416) --- .../compile/AliasingExpressionCompiler.java | 72 ------------------- .../phoenix/compile/DeleteCompiler.java | 2 +- .../phoenix/compile/FromCompiler.java | 25 +++++-- .../phoenix/compile/GroupByCompiler.java | 8 +-- .../phoenix/compile/OrderByCompiler.java | 7 +- .../phoenix/compile/ProjectionCompiler.java | 14 ---- .../phoenix/compile/QueryCompiler.java | 9 +-- .../salesforce/phoenix/compile/QueryPlan.java | 3 + .../phoenix/compile/StatementNormalizer.java | 16 ++++- ...rackOrderPreservingExpressionCompiler.java | 16 +++-- .../phoenix/execute/BasicQueryPlan.java | 20 +++++- .../IndexKeyValueColumnExpression.java | 11 ++- .../phoenix/jdbc/PhoenixDatabaseMetaData.java | 2 +- .../phoenix/optimize/QueryOptimizer.java | 4 +- .../phoenix/parse/ColumnParseNode.java | 13 ++++ .../phoenix/parse/ParseNodeRewriter.java | 55 +++++++++++++- .../parse/SelectStatementRewriter.java | 5 +- .../salesforce/phoenix/util/IndexUtil.java | 6 +- .../salesforce/phoenix/util/SchemaUtil.java | 4 +- .../phoenix/compile/HavingClauseTest.java | 27 ++++--- .../phoenix/compile/LimitClauseTest.java | 18 +++-- .../compile/SelectStatementRewriterTest.java | 12 +++- .../StatementHintsCompilationTest.java | 16 +++-- .../compile/WhereClauseFilterTest.java | 39 ++++++++-- .../compile/WhereClauseScanKeyTest.java | 35 ++++++--- .../phoenix/end2end/QueryExecTest.java | 3 +- .../end2end/index/MutableIndexTest.java | 14 +++- 27 files changed, 285 insertions(+), 171 deletions(-) delete mode 100644 src/main/java/com/salesforce/phoenix/compile/AliasingExpressionCompiler.java diff --git a/src/main/java/com/salesforce/phoenix/compile/AliasingExpressionCompiler.java b/src/main/java/com/salesforce/phoenix/compile/AliasingExpressionCompiler.java deleted file mode 100644 index 2c49a934..00000000 --- a/src/main/java/com/salesforce/phoenix/compile/AliasingExpressionCompiler.java +++ /dev/null @@ -1,72 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2013, Salesforce.com, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of Salesforce.com nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ******************************************************************************/ -package com.salesforce.phoenix.compile; - -import java.sql.SQLException; -import java.util.Map; - -import com.salesforce.phoenix.compile.GroupByCompiler.GroupBy; -import com.salesforce.phoenix.expression.Expression; -import com.salesforce.phoenix.parse.ColumnParseNode; -import com.salesforce.phoenix.parse.ParseNode; -import com.salesforce.phoenix.schema.ColumnNotFoundException; - -/** - * - * Expression compiler that may reference an expression through an alias - * - * @author jtaylor - * @since 1.2 - */ -public class AliasingExpressionCompiler extends ExpressionCompiler { - private final Map aliasParseNodeMap; - - AliasingExpressionCompiler(StatementContext context, Map aliasParseNodeMap) { - this(context, GroupBy.EMPTY_GROUP_BY, aliasParseNodeMap); - } - - protected AliasingExpressionCompiler(StatementContext context, GroupBy groupBy, Map aliasParseNodeMap) { - super(context, groupBy); - this.aliasParseNodeMap = aliasParseNodeMap; - } - - @Override - public Expression visit(ColumnParseNode node) throws SQLException { - try { - return super.visit(node); - } catch (ColumnNotFoundException e) { - // If we cannot find the column reference, check out alias map instead - ParseNode aliasedNode = aliasParseNodeMap.get(node.getName()); - if (aliasedNode != null) { // If we found an alias, in-line the parse nodes - return aliasedNode.accept(this); - } - throw e; - } - } - -} diff --git a/src/main/java/com/salesforce/phoenix/compile/DeleteCompiler.java b/src/main/java/com/salesforce/phoenix/compile/DeleteCompiler.java index bceb9a3d..7812aef3 100644 --- a/src/main/java/com/salesforce/phoenix/compile/DeleteCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/DeleteCompiler.java @@ -118,7 +118,7 @@ public MutationPlan compile(DeleteStatement statement, List binds) throw Scan scan = new Scan(); final StatementContext context = new StatementContext(statement, connection, resolver, binds, scan); final Integer limit = LimitCompiler.compile(context, statement); - final OrderBy orderBy = OrderByCompiler.compile(context, statement, Collections.emptyMap(), GroupBy.EMPTY_GROUP_BY, limit); + final OrderBy orderBy = OrderByCompiler.compile(context, statement, GroupBy.EMPTY_GROUP_BY, limit); Expression whereClause = WhereCompiler.compile(context, statement); final int maxSize = services.getProps().getInt(QueryServices.MAX_MUTATION_SIZE_ATTRIB,QueryServicesOptions.DEFAULT_MAX_MUTATION_SIZE); diff --git a/src/main/java/com/salesforce/phoenix/compile/FromCompiler.java b/src/main/java/com/salesforce/phoenix/compile/FromCompiler.java index 5dc984df..fa5a94df 100644 --- a/src/main/java/com/salesforce/phoenix/compile/FromCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/FromCompiler.java @@ -169,12 +169,27 @@ public List getTables() { public ColumnRef resolveColumn(String schemaName, String tableName, String colName) throws SQLException { TableRef tableRef = tableRefs.get(0); - if (schemaName != null && !schemaName.equals(alias)) { - throw new ColumnNotFoundException(schemaName, tableName, null, colName); + boolean resolveCF = false; + if (schemaName != null || tableName != null) { + String resolvedTableName = tableRef.getTable().getTableName().getString(); + String resolvedSchemaName = tableRef.getTable().getSchemaName().getString(); + if (schemaName != null && tableName != null) { + if ( ! ( schemaName.equals(resolvedSchemaName) && + tableName.equals(resolvedTableName) )) { + if (!(resolveCF = schemaName.equals(alias))) { + throw new ColumnNotFoundException(schemaName, tableName, null, colName); + } + } + } else { // schemaName == null && tableName != null + if (!tableName.equals(alias) && (!tableName.equals(resolvedTableName) || !resolvedSchemaName.equals(""))) { + resolveCF = true; + } + } + } - PColumn column = tableName == null || (schemaName == null && tableName.equals(alias)) ? - tableRef.getTable().getColumn(colName) : - tableRef.getTable().getColumnFamily(tableName).getColumn(colName); + PColumn column = resolveCF + ? tableRef.getTable().getColumnFamily(tableName).getColumn(colName) + : tableRef.getTable().getColumn(colName); return new ColumnRef(tableRef, column.getPosition()); } diff --git a/src/main/java/com/salesforce/phoenix/compile/GroupByCompiler.java b/src/main/java/com/salesforce/phoenix/compile/GroupByCompiler.java index a4bdfff6..49cf1e6d 100644 --- a/src/main/java/com/salesforce/phoenix/compile/GroupByCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/GroupByCompiler.java @@ -32,7 +32,6 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.Map; import org.apache.http.annotation.Immutable; @@ -142,12 +141,11 @@ public void explain(List planSteps) { * Get list of columns in the GROUP BY clause. * @param context query context kept between compilation of different query clauses * @param statement SQL statement being compiled - * @param aliasParseNodeMap map to resolve alias name references to parse node * @return the {@link GroupBy} instance encapsulating the group by clause * @throws ColumnNotFoundException if column name could not be resolved * @throws AmbiguousColumnException if an unaliased column name is ambiguous across multiple tables */ - public static GroupBy compile(StatementContext context, SelectStatement statement, Map aliasParseNodeMap) throws SQLException { + public static GroupBy compile(StatementContext context, SelectStatement statement) throws SQLException { List groupByNodes = statement.getGroupBy(); /** * Distinct can use an aggregate plan if there's no group by. @@ -171,8 +169,8 @@ public static GroupBy compile(StatementContext context, SelectStatement statemen // Accumulate expressions in GROUP BY TrackOrderPreservingExpressionCompiler groupByVisitor = new TrackOrderPreservingExpressionCompiler(context, - GroupBy.EMPTY_GROUP_BY, aliasParseNodeMap, - groupByNodes.size(), Ordering.UNORDERED); + GroupBy.EMPTY_GROUP_BY, groupByNodes.size(), + Ordering.UNORDERED); for (ParseNode node : groupByNodes) { Expression expression = node.accept(groupByVisitor); if (groupByVisitor.isAggregate()) { diff --git a/src/main/java/com/salesforce/phoenix/compile/OrderByCompiler.java b/src/main/java/com/salesforce/phoenix/compile/OrderByCompiler.java index cc4bf595..5f22441f 100644 --- a/src/main/java/com/salesforce/phoenix/compile/OrderByCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/OrderByCompiler.java @@ -71,8 +71,6 @@ public List getOrderByExpressions() { * @param context the query context for tracking various states * associated with the given select statement * @param statement TODO - * @param aliasMap the map of aliased parse nodes used - * to resolve alias usage in the ORDER BY clause * @param groupBy the list of columns in the GROUP BY clause * @param limit the row limit or null if no limit * @return the compiled ORDER BY clause @@ -80,8 +78,7 @@ public List getOrderByExpressions() { */ public static OrderBy compile(StatementContext context, FilterableStatement statement, - Map aliasMap, GroupBy groupBy, - Integer limit) throws SQLException { + GroupBy groupBy, Integer limit) throws SQLException { List orderByNodes = statement.getOrderBy(); if (orderByNodes.isEmpty()) { return OrderBy.EMPTY_ORDER_BY; @@ -89,7 +86,7 @@ public static OrderBy compile(StatementContext context, // accumulate columns in ORDER BY TrackOrderPreservingExpressionCompiler visitor = new TrackOrderPreservingExpressionCompiler(context, groupBy, - aliasMap, orderByNodes.size(), Ordering.ORDERED); + orderByNodes.size(), Ordering.ORDERED); LinkedHashSet orderByExpressions = Sets.newLinkedHashSetWithExpectedSize(orderByNodes.size()); for (OrderByNode node : orderByNodes) { boolean isAscending = node.isAscending(); diff --git a/src/main/java/com/salesforce/phoenix/compile/ProjectionCompiler.java b/src/main/java/com/salesforce/phoenix/compile/ProjectionCompiler.java index 07c52a22..3c21f50e 100644 --- a/src/main/java/com/salesforce/phoenix/compile/ProjectionCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/ProjectionCompiler.java @@ -41,7 +41,6 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.salesforce.phoenix.compile.GroupByCompiler.GroupBy; import com.salesforce.phoenix.coprocessor.GroupedAggregateRegionObserver; @@ -98,18 +97,6 @@ private static void projectColumnFamily(PTable table, Scan scan, byte[] family) scan.addFamily(family); } - public static Map buildAliasMap(StatementContext context, SelectStatement statement) { - List aliasedNodes = statement.getSelect(); - Map aliasParseNodeMap = Maps.newHashMapWithExpectedSize(aliasedNodes.size()); - for (AliasedNode aliasedNode : aliasedNodes) { - String alias = aliasedNode.getAlias(); - if (alias != null) { - aliasParseNodeMap.put(alias, aliasedNode.getNode()); - } - } - return aliasParseNodeMap; - } - public static RowProjector compile(StatementContext context, SelectStatement statement, GroupBy groupBy) throws SQLException { return compile(context, statement, groupBy, null); } @@ -192,7 +179,6 @@ public static RowProjector compile(StatementContext context, SelectStatement sta int index = 0; List projectedExpressions = Lists.newArrayListWithExpectedSize(aliasedNodes.size()); List projectedFamilies = Lists.newArrayListWithExpectedSize(aliasedNodes.size()); - // TODO: support cf.* expressions in projection to project all columns in a CF for (AliasedNode aliasedNode : aliasedNodes) { ParseNode node = aliasedNode.getNode(); // TODO: visitor? diff --git a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java index f5fe3632..e28f7935 100644 --- a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java @@ -30,7 +30,6 @@ import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; import java.util.List; -import java.util.Map; import org.apache.hadoop.hbase.client.Scan; @@ -44,7 +43,6 @@ import com.salesforce.phoenix.iterate.SpoolingResultIterator.SpoolingResultIteratorFactory; import com.salesforce.phoenix.jdbc.PhoenixConnection; import com.salesforce.phoenix.jdbc.PhoenixDatabaseMetaData; -import com.salesforce.phoenix.parse.ParseNode; import com.salesforce.phoenix.parse.SelectStatement; import com.salesforce.phoenix.query.QueryConstants; import com.salesforce.phoenix.schema.AmbiguousColumnException; @@ -116,8 +114,8 @@ public QueryCompiler(PhoenixConnection connection, int maxRows, Scan scan, PColu public QueryPlan compile(SelectStatement statement, List binds) throws SQLException { assert(binds.size() == statement.getBindCount()); - statement = StatementNormalizer.normalize(statement); ColumnResolver resolver = FromCompiler.getResolver(statement, connection); + statement = StatementNormalizer.normalize(statement, resolver); TableRef tableRef = resolver.getTables().get(0); StatementContext context = new StatementContext(statement, connection, resolver, binds, scan); // Short circuit out if we're compiling an index query and the index isn't active. @@ -126,10 +124,9 @@ public QueryPlan compile(SelectStatement statement, List binds) throws S if (tableRef.getTable().getType() == PTableType.INDEX && tableRef.getTable().getIndexState() != PIndexState.ACTIVE) { return new DegenerateQueryPlan(context, statement, tableRef); } - Map aliasMap = ProjectionCompiler.buildAliasMap(context, statement); Integer limit = LimitCompiler.compile(context, statement); - GroupBy groupBy = GroupByCompiler.compile(context, statement, aliasMap); + GroupBy groupBy = GroupByCompiler.compile(context, statement); // Optimize the HAVING clause by finding any group by expressions that can be moved // to the WHERE clause statement = HavingCompiler.rewrite(context, statement, groupBy); @@ -137,7 +134,7 @@ public QueryPlan compile(SelectStatement statement, List binds) throws S // Don't pass groupBy when building where clause expression, because we do not want to wrap these // expressions as group by key expressions since they're pre, not post filtered. WhereCompiler.compile(context, statement); - OrderBy orderBy = OrderByCompiler.compile(context, statement, aliasMap, groupBy, limit); + OrderBy orderBy = OrderByCompiler.compile(context, statement, groupBy, limit); RowProjector projector = ProjectionCompiler.compile(context, statement, groupBy, targetColumns); // Final step is to build the query plan diff --git a/src/main/java/com/salesforce/phoenix/compile/QueryPlan.java b/src/main/java/com/salesforce/phoenix/compile/QueryPlan.java index 465be2cd..94f81217 100644 --- a/src/main/java/com/salesforce/phoenix/compile/QueryPlan.java +++ b/src/main/java/com/salesforce/phoenix/compile/QueryPlan.java @@ -32,6 +32,7 @@ import com.salesforce.phoenix.compile.GroupByCompiler.GroupBy; import com.salesforce.phoenix.compile.OrderByCompiler.OrderBy; +import com.salesforce.phoenix.parse.FilterableStatement; import com.salesforce.phoenix.query.KeyRange; import com.salesforce.phoenix.query.Scanner; import com.salesforce.phoenix.schema.TableRef; @@ -69,4 +70,6 @@ public interface QueryPlan extends StatementPlan { List getSplits(); StatementContext getContext(); + + FilterableStatement getStatement(); } diff --git a/src/main/java/com/salesforce/phoenix/compile/StatementNormalizer.java b/src/main/java/com/salesforce/phoenix/compile/StatementNormalizer.java index 803b2750..4cf37090 100644 --- a/src/main/java/com/salesforce/phoenix/compile/StatementNormalizer.java +++ b/src/main/java/com/salesforce/phoenix/compile/StatementNormalizer.java @@ -31,7 +31,12 @@ import java.util.List; import com.google.common.collect.Lists; -import com.salesforce.phoenix.parse.*; +import com.salesforce.phoenix.parse.BetweenParseNode; +import com.salesforce.phoenix.parse.ComparisonParseNode; +import com.salesforce.phoenix.parse.LessThanOrEqualParseNode; +import com.salesforce.phoenix.parse.ParseNode; +import com.salesforce.phoenix.parse.ParseNodeRewriter; +import com.salesforce.phoenix.parse.SelectStatement; /** @@ -45,15 +50,20 @@ */ public class StatementNormalizer extends ParseNodeRewriter { + public StatementNormalizer(ColumnResolver resolver, int expectedAliasCount) { + super(resolver, expectedAliasCount); + } + /** * Rewrite the select statement by switching any constants to the right hand side * of the expression. * @param statement the select statement + * @param resolver * @return new select statement * @throws SQLException */ - public static SelectStatement normalize(SelectStatement statement) throws SQLException { - return rewrite(statement, new StatementNormalizer()); + public static SelectStatement normalize(SelectStatement statement, ColumnResolver resolver) throws SQLException { + return rewrite(statement, new StatementNormalizer(resolver, statement.getSelect().size())); } @Override diff --git a/src/main/java/com/salesforce/phoenix/compile/TrackOrderPreservingExpressionCompiler.java b/src/main/java/com/salesforce/phoenix/compile/TrackOrderPreservingExpressionCompiler.java index 6883d0b2..2b594c89 100644 --- a/src/main/java/com/salesforce/phoenix/compile/TrackOrderPreservingExpressionCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/TrackOrderPreservingExpressionCompiler.java @@ -1,7 +1,9 @@ package com.salesforce.phoenix.compile; import java.sql.SQLException; -import java.util.*; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; import com.google.common.base.Objects; import com.google.common.collect.Lists; @@ -10,7 +12,11 @@ import com.salesforce.phoenix.expression.LiteralExpression; import com.salesforce.phoenix.expression.function.FunctionExpression; import com.salesforce.phoenix.expression.function.FunctionExpression.OrderPreserving; -import com.salesforce.phoenix.parse.*; +import com.salesforce.phoenix.parse.CaseParseNode; +import com.salesforce.phoenix.parse.ColumnParseNode; +import com.salesforce.phoenix.parse.DivideParseNode; +import com.salesforce.phoenix.parse.MultiplyParseNode; +import com.salesforce.phoenix.parse.SubtractParseNode; import com.salesforce.phoenix.schema.ColumnModifier; import com.salesforce.phoenix.schema.ColumnRef; import com.salesforce.phoenix.util.SchemaUtil; @@ -25,7 +31,7 @@ * the order is preserved. * */ -public class TrackOrderPreservingExpressionCompiler extends AliasingExpressionCompiler { +public class TrackOrderPreservingExpressionCompiler extends ExpressionCompiler { public enum Ordering {ORDERED, UNORDERED}; private final List entries; @@ -34,8 +40,8 @@ public enum Ordering {ORDERED, UNORDERED}; private ColumnRef columnRef; private boolean isOrderPreserving = true; - TrackOrderPreservingExpressionCompiler(StatementContext context, GroupBy groupBy, Map aliasParseNodeMap, int expectedEntrySize, Ordering ordering) { - super(context, groupBy, aliasParseNodeMap); + TrackOrderPreservingExpressionCompiler(StatementContext context, GroupBy groupBy, int expectedEntrySize, Ordering ordering) { + super(context, groupBy); if (context.getResolver().getTables().get(0).getTable().getBucketNum() != null) { orderPreserving = OrderPreserving.NO; } diff --git a/src/main/java/com/salesforce/phoenix/execute/BasicQueryPlan.java b/src/main/java/com/salesforce/phoenix/execute/BasicQueryPlan.java index 55ef61c4..bdb1584b 100644 --- a/src/main/java/com/salesforce/phoenix/execute/BasicQueryPlan.java +++ b/src/main/java/com/salesforce/phoenix/execute/BasicQueryPlan.java @@ -33,14 +33,23 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import com.salesforce.phoenix.compile.*; +import com.salesforce.phoenix.compile.ExplainPlan; import com.salesforce.phoenix.compile.GroupByCompiler.GroupBy; import com.salesforce.phoenix.compile.OrderByCompiler.OrderBy; +import com.salesforce.phoenix.compile.QueryPlan; +import com.salesforce.phoenix.compile.RowProjector; +import com.salesforce.phoenix.compile.ScanRanges; +import com.salesforce.phoenix.compile.StatementContext; import com.salesforce.phoenix.iterate.ParallelIterators.ParallelIteratorFactory; import com.salesforce.phoenix.jdbc.PhoenixConnection; import com.salesforce.phoenix.parse.FilterableStatement; -import com.salesforce.phoenix.query.*; -import com.salesforce.phoenix.schema.*; +import com.salesforce.phoenix.query.ConnectionQueryServices; +import com.salesforce.phoenix.query.DegenerateScanner; +import com.salesforce.phoenix.query.QueryConstants; +import com.salesforce.phoenix.query.Scanner; +import com.salesforce.phoenix.schema.PTable; +import com.salesforce.phoenix.schema.PTableType; +import com.salesforce.phoenix.schema.TableRef; import com.salesforce.phoenix.util.ScanUtil; import com.salesforce.phoenix.util.SchemaUtil; @@ -167,6 +176,11 @@ public ParameterMetaData getParameterMetaData() { return paramMetaData; } + @Override + public FilterableStatement getStatement() { + return statement; + } + @Override public StatementContext getContext() { return context; diff --git a/src/main/java/com/salesforce/phoenix/expression/IndexKeyValueColumnExpression.java b/src/main/java/com/salesforce/phoenix/expression/IndexKeyValueColumnExpression.java index c119389a..65a08039 100644 --- a/src/main/java/com/salesforce/phoenix/expression/IndexKeyValueColumnExpression.java +++ b/src/main/java/com/salesforce/phoenix/expression/IndexKeyValueColumnExpression.java @@ -1,10 +1,12 @@ package com.salesforce.phoenix.expression; +import org.apache.hadoop.hbase.util.Bytes; + import com.salesforce.phoenix.schema.PColumn; +import com.salesforce.phoenix.util.IndexUtil; import com.salesforce.phoenix.util.SchemaUtil; public class IndexKeyValueColumnExpression extends KeyValueColumnExpression { - public IndexKeyValueColumnExpression() { } @@ -14,8 +16,11 @@ public IndexKeyValueColumnExpression(PColumn column) { @Override public String toString() { - // Just display the column name part, since it's guaranteed to be unique - return SchemaUtil.getColumnDisplayName(this.getColumnName()); + // Translate to the data table column name + String indexColumnName = Bytes.toString(this.getColumnName()); + String dataFamilyName = IndexUtil.getDataColumnFamilyName(indexColumnName); + String dataColumnName = IndexUtil.getDataColumnName(indexColumnName); + return SchemaUtil.getColumnDisplayName(dataFamilyName, dataColumnName); } } diff --git a/src/main/java/com/salesforce/phoenix/jdbc/PhoenixDatabaseMetaData.java b/src/main/java/com/salesforce/phoenix/jdbc/PhoenixDatabaseMetaData.java index 6fdc73cc..7f44b767 100644 --- a/src/main/java/com/salesforce/phoenix/jdbc/PhoenixDatabaseMetaData.java +++ b/src/main/java/com/salesforce/phoenix/jdbc/PhoenixDatabaseMetaData.java @@ -865,7 +865,7 @@ public ResultSet getTables(String catalog, String schemaPattern, String tableNam } buf.setCharAt(buf.length()-1, ')'); } - buf.append(" order by " + TABLE_TYPE_NAME + "," + TABLE_SCHEM_NAME + "," + TABLE_NAME_NAME); + buf.append(" order by " + TYPE_SCHEMA_AND_TABLE + "." +TABLE_TYPE_NAME + "," + TABLE_SCHEM_NAME + "," + TABLE_NAME_NAME); Statement stmt = connection.createStatement(); return stmt.executeQuery(buf.toString()); } diff --git a/src/main/java/com/salesforce/phoenix/optimize/QueryOptimizer.java b/src/main/java/com/salesforce/phoenix/optimize/QueryOptimizer.java index 003e2dcb..f64ec949 100644 --- a/src/main/java/com/salesforce/phoenix/optimize/QueryOptimizer.java +++ b/src/main/java/com/salesforce/phoenix/optimize/QueryOptimizer.java @@ -44,7 +44,9 @@ public QueryPlan optimize(SelectStatement select, PhoenixStatement statement) th if (!useIndexes) { return dataPlan; } - + // Get the statement as it's been normalized now + // TODO: the recompile for the index tables could skip the normalize step + select = (SelectStatement)dataPlan.getStatement(); PTable dataTable = dataPlan.getTableRef().getTable(); Listindexes = Lists.newArrayList(dataTable.getIndexes()); if (indexes.isEmpty() || dataPlan.getTableRef().hasDynamicCols() || select.getHint().hasHint(Hint.NO_INDEX)) { diff --git a/src/main/java/com/salesforce/phoenix/parse/ColumnParseNode.java b/src/main/java/com/salesforce/phoenix/parse/ColumnParseNode.java index 0c34d674..69961c9a 100644 --- a/src/main/java/com/salesforce/phoenix/parse/ColumnParseNode.java +++ b/src/main/java/com/salesforce/phoenix/parse/ColumnParseNode.java @@ -69,4 +69,17 @@ public String toString() { return alias == null ? getName() : alias; } + @Override + public int hashCode() { + return fullName.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + ColumnParseNode other = (ColumnParseNode)obj; + return fullName.equals(other.fullName); + } } diff --git a/src/main/java/com/salesforce/phoenix/parse/ParseNodeRewriter.java b/src/main/java/com/salesforce/phoenix/parse/ParseNodeRewriter.java index e00cacb6..2485f390 100644 --- a/src/main/java/com/salesforce/phoenix/parse/ParseNodeRewriter.java +++ b/src/main/java/com/salesforce/phoenix/parse/ParseNodeRewriter.java @@ -30,8 +30,13 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.salesforce.phoenix.compile.ColumnResolver; +import com.salesforce.phoenix.schema.AmbiguousColumnException; +import com.salesforce.phoenix.schema.ColumnNotFoundException; /** * @@ -53,6 +58,7 @@ public class ParseNodeRewriter extends TraverseAllParseNodeVisitor { * @throws SQLException */ public static SelectStatement rewrite(SelectStatement statement, ParseNodeRewriter rewriter) throws SQLException { + Map aliasMap = rewriter.getAliasMap(); ParseNode where = statement.getWhere(); ParseNode normWhere = where; if (where != null) { @@ -81,8 +87,22 @@ public static SelectStatement rewrite(SelectStatement statement, ParseNodeRewrit if (selectNodes == normSelectNodes) { normSelectNodes = Lists.newArrayList(selectNodes.subList(0, i)); } - normSelectNodes.add(NODE_FACTORY.aliasedNode(aliasedNode.getAlias(), normSelectNode)); + AliasedNode normAliasNode = NODE_FACTORY.aliasedNode(aliasedNode.getAlias(), normSelectNode); + normSelectNodes.add(normAliasNode); } + // Add to map in separate pass so that we don't try to use aliases + // while processing the select expressions + if (aliasMap != null) { + for (int i = 0; i < normSelectNodes.size(); i++) { + AliasedNode aliasedNode = normSelectNodes.get(i); + ParseNode selectNode = aliasedNode.getNode(); + String alias = aliasedNode.getAlias(); + if (alias != null) { + aliasMap.put(alias, selectNode); + } + } + } + List groupByNodes = statement.getGroupBy(); List normGroupByNodes = groupByNodes; for (int i = 0; i < groupByNodes.size(); i++) { @@ -131,6 +151,23 @@ public static SelectStatement rewrite(SelectStatement statement, ParseNodeRewrit statement.getLimit(), statement.getBindCount(), statement.isAggregate()); } + private Map getAliasMap() { + return aliasMap; + } + + private final ColumnResolver resolver; + private final Map aliasMap; + + protected ParseNodeRewriter() { + aliasMap = null; + resolver = null; + } + + protected ParseNodeRewriter(ColumnResolver resolver, int maxAliasCount) { + this.resolver = resolver; + aliasMap = Maps.newHashMapWithExpectedSize(maxAliasCount); + } + protected void reset() { } @@ -302,6 +339,22 @@ public ParseNode createNode(List children) { @Override public ParseNode visit(ColumnParseNode node) throws SQLException { + // If we're resolving aliases and we have an unqualified ColumnParseNode, + // check if we find the name in our alias map. + if (aliasMap != null && node.getTableName() == null) { + ParseNode aliasedNode = aliasMap.get(node.getName()); + // If we found something, then try to resolve it unless the two nodes are the same + if (aliasedNode != null && !node.equals(aliasedNode)) { + try { + // If we're able to resolve it, that means we have a conflict + resolver.resolveColumn(node.getSchemaName(), node.getTableName(), node.getName()); + throw new AmbiguousColumnException(node.getName()); + } catch (ColumnNotFoundException e) { + // Not able to resolve alias as a column name as well, so we use the alias + return aliasedNode; + } + } + } return node; } diff --git a/src/main/java/com/salesforce/phoenix/parse/SelectStatementRewriter.java b/src/main/java/com/salesforce/phoenix/parse/SelectStatementRewriter.java index 6d33a432..d79f67f4 100644 --- a/src/main/java/com/salesforce/phoenix/parse/SelectStatementRewriter.java +++ b/src/main/java/com/salesforce/phoenix/parse/SelectStatementRewriter.java @@ -28,7 +28,10 @@ package com.salesforce.phoenix.parse; import java.sql.SQLException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; diff --git a/src/main/java/com/salesforce/phoenix/util/IndexUtil.java b/src/main/java/com/salesforce/phoenix/util/IndexUtil.java index 7e3a86c8..83f9626e 100644 --- a/src/main/java/com/salesforce/phoenix/util/IndexUtil.java +++ b/src/main/java/com/salesforce/phoenix/util/IndexUtil.java @@ -88,7 +88,11 @@ public static PDataType getIndexColumnDataType(boolean isNullable, PDataType dat public static String getDataColumnName(String name) { - return name.substring(name.indexOf(INDEX_COLUMN_NAME_SEP)); + return name.substring(name.indexOf(INDEX_COLUMN_NAME_SEP) + 1); + } + + public static String getDataColumnFamilyName(String name) { + return name.substring(0,name.indexOf(INDEX_COLUMN_NAME_SEP)); } public static String getIndexColumnName(String dataColumnFamilyName, String dataColumnName) { diff --git a/src/main/java/com/salesforce/phoenix/util/SchemaUtil.java b/src/main/java/com/salesforce/phoenix/util/SchemaUtil.java index 39a909e7..06b1ea2b 100644 --- a/src/main/java/com/salesforce/phoenix/util/SchemaUtil.java +++ b/src/main/java/com/salesforce/phoenix/util/SchemaUtil.java @@ -275,8 +275,8 @@ public static String getColumnDisplayName(byte[] cf, byte[] cq) { return getName(Bytes.compareTo(cf, QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES) == 0 ? ByteUtil.EMPTY_BYTE_ARRAY : cf, cq); } - public static String getColumnDisplayName(byte[] cq) { - return getName(ByteUtil.EMPTY_BYTE_ARRAY, cq); + public static String getColumnDisplayName(String cf, String cq) { + return getName(QueryConstants.DEFAULT_COLUMN_FAMILY.equals(cf) ? null : cf, cq); } public static String getMetaDataEntityName(String schemaName, String tableName, String familyName, String columnName) { diff --git a/src/test/java/com/salesforce/phoenix/compile/HavingClauseTest.java b/src/test/java/com/salesforce/phoenix/compile/HavingClauseTest.java index 815e2570..5069de90 100644 --- a/src/test/java/com/salesforce/phoenix/compile/HavingClauseTest.java +++ b/src/test/java/com/salesforce/phoenix/compile/HavingClauseTest.java @@ -27,12 +27,23 @@ ******************************************************************************/ package com.salesforce.phoenix.compile; -import static com.salesforce.phoenix.util.TestUtil.*; -import static org.junit.Assert.*; +import static com.salesforce.phoenix.util.TestUtil.TEST_PROPERTIES; +import static com.salesforce.phoenix.util.TestUtil.and; +import static com.salesforce.phoenix.util.TestUtil.constantComparison; +import static com.salesforce.phoenix.util.TestUtil.kvColumn; +import static com.salesforce.phoenix.util.TestUtil.or; +import static com.salesforce.phoenix.util.TestUtil.pkColumn; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; -import java.sql.*; import java.sql.Date; -import java.util.*; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; @@ -44,7 +55,8 @@ import com.salesforce.phoenix.expression.function.CountAggregateFunction; import com.salesforce.phoenix.expression.function.RoundFunction; import com.salesforce.phoenix.jdbc.PhoenixConnection; -import com.salesforce.phoenix.parse.*; +import com.salesforce.phoenix.parse.SQLParser; +import com.salesforce.phoenix.parse.SelectStatement; import com.salesforce.phoenix.query.BaseConnectionlessQueryTest; @@ -65,13 +77,12 @@ private static Expressions compileStatement(String query, List binds) th SQLParser parser = new SQLParser(query); SelectStatement statement = parser.parseQuery(); Scan scan = new Scan(); - statement = StatementNormalizer.normalize(statement); - Map aliasParseNodeMap = ProjectionCompiler.buildAliasMap(context, statement); PhoenixConnection pconn = DriverManager.getConnection(getUrl(), TEST_PROPERTIES).unwrap(PhoenixConnection.class); ColumnResolver resolver = FromCompiler.getResolver(statement, pconn); + statement = StatementNormalizer.normalize(statement, resolver); context = new StatementContext(statement, pconn, resolver, binds, scan); - GroupBy groupBy = GroupByCompiler.compile(context, statement, aliasParseNodeMap); + GroupBy groupBy = GroupByCompiler.compile(context, statement); // Optimize the HAVING clause by finding any group by expressions that can be moved // to the WHERE clause statement = HavingCompiler.rewrite(context, statement, groupBy); diff --git a/src/test/java/com/salesforce/phoenix/compile/LimitClauseTest.java b/src/test/java/com/salesforce/phoenix/compile/LimitClauseTest.java index 76127dae..1fb815de 100644 --- a/src/test/java/com/salesforce/phoenix/compile/LimitClauseTest.java +++ b/src/test/java/com/salesforce/phoenix/compile/LimitClauseTest.java @@ -28,11 +28,17 @@ package com.salesforce.phoenix.compile; import static com.salesforce.phoenix.util.TestUtil.TEST_PROPERTIES; -import static org.junit.Assert.*; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.sql.DriverManager; import java.sql.SQLException; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import org.apache.hadoop.hbase.client.Scan; import org.junit.Test; @@ -40,7 +46,8 @@ import com.salesforce.phoenix.compile.GroupByCompiler.GroupBy; import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.jdbc.PhoenixConnection; -import com.salesforce.phoenix.parse.*; +import com.salesforce.phoenix.parse.SQLParser; +import com.salesforce.phoenix.parse.SelectStatement; import com.salesforce.phoenix.query.BaseConnectionlessQueryTest; import com.salesforce.phoenix.schema.PDataType; import com.salesforce.phoenix.util.ByteUtil; @@ -51,14 +58,13 @@ public class LimitClauseTest extends BaseConnectionlessQueryTest { private static Integer compileStatement(String query, List binds, Scan scan) throws SQLException { SQLParser parser = new SQLParser(query); SelectStatement statement = parser.parseQuery(); - statement = StatementNormalizer.normalize(statement); PhoenixConnection pconn = DriverManager.getConnection(getUrl(), TEST_PROPERTIES).unwrap(PhoenixConnection.class); ColumnResolver resolver = FromCompiler.getResolver(statement, pconn); + statement = StatementNormalizer.normalize(statement, resolver); StatementContext context = new StatementContext(statement, pconn, resolver, binds, scan); - Map aliasParseNodeMap = ProjectionCompiler.buildAliasMap(context, statement); Integer limit = LimitCompiler.compile(context, statement); - GroupBy groupBy = GroupByCompiler.compile(context, statement, aliasParseNodeMap); + GroupBy groupBy = GroupByCompiler.compile(context, statement); statement = HavingCompiler.rewrite(context, statement, groupBy); HavingCompiler.compile(context, statement, groupBy); Expression where = WhereCompiler.compile(context, statement); diff --git a/src/test/java/com/salesforce/phoenix/compile/SelectStatementRewriterTest.java b/src/test/java/com/salesforce/phoenix/compile/SelectStatementRewriterTest.java index 9b1e5917..5ecf4fb4 100644 --- a/src/test/java/com/salesforce/phoenix/compile/SelectStatementRewriterTest.java +++ b/src/test/java/com/salesforce/phoenix/compile/SelectStatementRewriterTest.java @@ -28,7 +28,9 @@ package com.salesforce.phoenix.compile; import static com.salesforce.phoenix.util.TestUtil.TEST_PROPERTIES; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import java.sql.DriverManager; import java.sql.SQLException; @@ -39,7 +41,11 @@ import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.junit.Test; -import com.salesforce.phoenix.expression.*; +import com.salesforce.phoenix.expression.AndExpression; +import com.salesforce.phoenix.expression.ComparisonExpression; +import com.salesforce.phoenix.expression.Expression; +import com.salesforce.phoenix.expression.KeyValueColumnExpression; +import com.salesforce.phoenix.expression.LiteralExpression; import com.salesforce.phoenix.jdbc.PhoenixConnection; import com.salesforce.phoenix.parse.SQLParser; import com.salesforce.phoenix.parse.SelectStatement; @@ -55,8 +61,8 @@ private static Expression compileStatement(String query) throws SQLException { SelectStatement statement = parser.parseQuery(); PhoenixConnection pconn = DriverManager.getConnection(getUrl(), TEST_PROPERTIES).unwrap(PhoenixConnection.class); ColumnResolver resolver = FromCompiler.getResolver(statement, pconn); + statement = StatementNormalizer.normalize(statement, resolver); StatementContext context = new StatementContext(statement, pconn, resolver, binds, scan); - statement = StatementNormalizer.normalize(statement); Expression whereClause = WhereCompiler.compile(context, statement); return WhereOptimizer.pushKeyExpressionsToScan(context, statement, whereClause); } diff --git a/src/test/java/com/salesforce/phoenix/compile/StatementHintsCompilationTest.java b/src/test/java/com/salesforce/phoenix/compile/StatementHintsCompilationTest.java index 018cb8fc..105f5eba 100644 --- a/src/test/java/com/salesforce/phoenix/compile/StatementHintsCompilationTest.java +++ b/src/test/java/com/salesforce/phoenix/compile/StatementHintsCompilationTest.java @@ -28,11 +28,15 @@ package com.salesforce.phoenix.compile; import static com.salesforce.phoenix.util.TestUtil.TEST_PROPERTIES; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import java.sql.DriverManager; import java.sql.SQLException; -import java.util.*; +import java.util.Collections; +import java.util.List; +import java.util.Set; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.Filter; @@ -43,7 +47,8 @@ import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.filter.SkipScanFilter; import com.salesforce.phoenix.jdbc.PhoenixConnection; -import com.salesforce.phoenix.parse.*; +import com.salesforce.phoenix.parse.SQLParser; +import com.salesforce.phoenix.parse.SelectStatement; import com.salesforce.phoenix.query.BaseConnectionlessQueryTest; @@ -59,15 +64,14 @@ private static StatementContext compileStatement(String query, Scan scan, List binds, Integer limit, Set extractedNodes) throws SQLException { SQLParser parser = new SQLParser(query); SelectStatement statement = parser.parseQuery(); - statement = StatementNormalizer.normalize(statement); PhoenixConnection pconn = DriverManager.getConnection(getUrl(), TEST_PROPERTIES).unwrap(PhoenixConnection.class); ColumnResolver resolver = FromCompiler.getResolver(statement, pconn); + statement = StatementNormalizer.normalize(statement, resolver); StatementContext context = new StatementContext(statement, pconn, resolver, binds, scan); - Map aliasParseNodeMap = ProjectionCompiler.buildAliasMap(context, statement); Integer actualLimit = LimitCompiler.compile(context, statement); assertEquals(limit, actualLimit); - GroupBy groupBy = GroupByCompiler.compile(context, statement, aliasParseNodeMap); + GroupBy groupBy = GroupByCompiler.compile(context, statement); statement = HavingCompiler.rewrite(context, statement, groupBy); WhereCompiler.compileWhereClause(context, statement, extractedNodes); return context; diff --git a/src/test/java/com/salesforce/phoenix/compile/WhereClauseFilterTest.java b/src/test/java/com/salesforce/phoenix/compile/WhereClauseFilterTest.java index 86c84dc2..b96b4e19 100644 --- a/src/test/java/com/salesforce/phoenix/compile/WhereClauseFilterTest.java +++ b/src/test/java/com/salesforce/phoenix/compile/WhereClauseFilterTest.java @@ -27,15 +27,34 @@ ******************************************************************************/ package com.salesforce.phoenix.compile; -import static com.salesforce.phoenix.util.TestUtil.*; +import static com.salesforce.phoenix.util.TestUtil.ATABLE_NAME; +import static com.salesforce.phoenix.util.TestUtil.TEST_PROPERTIES; +import static com.salesforce.phoenix.util.TestUtil.and; +import static com.salesforce.phoenix.util.TestUtil.assertDegenerate; +import static com.salesforce.phoenix.util.TestUtil.columnComparison; +import static com.salesforce.phoenix.util.TestUtil.constantComparison; +import static com.salesforce.phoenix.util.TestUtil.in; +import static com.salesforce.phoenix.util.TestUtil.kvColumn; +import static com.salesforce.phoenix.util.TestUtil.multiKVFilter; +import static com.salesforce.phoenix.util.TestUtil.not; +import static com.salesforce.phoenix.util.TestUtil.or; +import static com.salesforce.phoenix.util.TestUtil.singleKVFilter; import static java.util.Collections.emptyList; -import static org.junit.Assert.*; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.math.BigDecimal; import java.sql.DriverManager; import java.sql.SQLException; import java.text.Format; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; @@ -46,23 +65,29 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; -import com.salesforce.phoenix.expression.*; +import com.salesforce.phoenix.expression.Expression; +import com.salesforce.phoenix.expression.LiteralExpression; +import com.salesforce.phoenix.expression.RowKeyColumnExpression; import com.salesforce.phoenix.expression.function.SubstrFunction; import com.salesforce.phoenix.filter.RowKeyComparisonFilter; import com.salesforce.phoenix.filter.SkipScanFilter; import com.salesforce.phoenix.jdbc.PhoenixConnection; import com.salesforce.phoenix.parse.SQLParser; import com.salesforce.phoenix.parse.SelectStatement; -import com.salesforce.phoenix.query.*; +import com.salesforce.phoenix.query.BaseConnectionlessQueryTest; +import com.salesforce.phoenix.query.KeyRange; +import com.salesforce.phoenix.query.QueryConstants; import com.salesforce.phoenix.schema.PDataType; import com.salesforce.phoenix.schema.RowKeyValueAccessor; -import com.salesforce.phoenix.util.*; +import com.salesforce.phoenix.util.ByteUtil; +import com.salesforce.phoenix.util.DateUtil; +import com.salesforce.phoenix.util.NumberUtil; public class WhereClauseFilterTest extends BaseConnectionlessQueryTest { private static SelectStatement compileStatement(StatementContext context, SelectStatement statement, ColumnResolver resolver, List binds, Scan scan, Integer expectedExtractedNodesSize, Integer expectedLimit) throws SQLException { - statement = StatementNormalizer.normalize(statement); + statement = StatementNormalizer.normalize(statement, resolver); Integer limit = LimitCompiler.compile(context, statement); assertEquals(expectedLimit, limit); diff --git a/src/test/java/com/salesforce/phoenix/compile/WhereClauseScanKeyTest.java b/src/test/java/com/salesforce/phoenix/compile/WhereClauseScanKeyTest.java index 374ef10b..a55c463a 100644 --- a/src/test/java/com/salesforce/phoenix/compile/WhereClauseScanKeyTest.java +++ b/src/test/java/com/salesforce/phoenix/compile/WhereClauseScanKeyTest.java @@ -27,12 +27,25 @@ ******************************************************************************/ package com.salesforce.phoenix.compile; -import static com.salesforce.phoenix.util.TestUtil.*; -import static org.junit.Assert.*; - -import java.sql.*; +import static com.salesforce.phoenix.util.TestUtil.TEST_PROPERTIES; +import static com.salesforce.phoenix.util.TestUtil.assertDegenerate; +import static com.salesforce.phoenix.util.TestUtil.assertEmptyScanKey; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.sql.Connection; import java.sql.Date; -import java.util.*; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.Scan; @@ -45,8 +58,11 @@ import com.salesforce.phoenix.filter.RowKeyComparisonFilter; import com.salesforce.phoenix.filter.SkipScanFilter; import com.salesforce.phoenix.jdbc.PhoenixConnection; -import com.salesforce.phoenix.parse.*; -import com.salesforce.phoenix.query.*; +import com.salesforce.phoenix.parse.SQLParser; +import com.salesforce.phoenix.parse.SelectStatement; +import com.salesforce.phoenix.query.BaseConnectionlessQueryTest; +import com.salesforce.phoenix.query.KeyRange; +import com.salesforce.phoenix.query.QueryConstants; import com.salesforce.phoenix.schema.ColumnNotFoundException; import com.salesforce.phoenix.schema.PDataType; import com.salesforce.phoenix.util.ByteUtil; @@ -71,15 +87,14 @@ private static StatementContext compileStatement(String query, Scan scan, List binds, Integer limit, Set extractedNodes) throws SQLException { SQLParser parser = new SQLParser(query); SelectStatement statement = parser.parseQuery(); - statement = StatementNormalizer.normalize(statement); PhoenixConnection pconn = DriverManager.getConnection(getUrl(), TEST_PROPERTIES).unwrap(PhoenixConnection.class); ColumnResolver resolver = FromCompiler.getResolver(statement, pconn); + statement = StatementNormalizer.normalize(statement, resolver); StatementContext context = new StatementContext(statement, pconn, resolver, binds, scan); - Map aliasParseNodeMap = ProjectionCompiler.buildAliasMap(context, statement); Integer actualLimit = LimitCompiler.compile(context, statement); assertEquals(limit, actualLimit); - GroupBy groupBy = GroupByCompiler.compile(context, statement, aliasParseNodeMap); + GroupBy groupBy = GroupByCompiler.compile(context, statement); statement = HavingCompiler.rewrite(context, statement, groupBy); WhereCompiler.compileWhereClause(context, statement, extractedNodes); return context; diff --git a/src/test/java/com/salesforce/phoenix/end2end/QueryExecTest.java b/src/test/java/com/salesforce/phoenix/end2end/QueryExecTest.java index 313741fb..a87f9cbd 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/QueryExecTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/QueryExecTest.java @@ -1080,7 +1080,8 @@ public void testGroupedAggregation() throws Exception { long ts = nextTimestamp(); String tenantId = getOrganizationId(); initATableValues(tenantId, getDefaultSplits(tenantId), null, ts); - String query = "SELECT a_string, count(1), 'foo' FROM atable WHERE organization_id=? GROUP BY a_string"; + // Tests that you don't get an ambiguous column exception when using the same alias as the column name + String query = "SELECT a_string as a_string, count(1), 'foo' FROM atable WHERE organization_id=? GROUP BY a_string"; Properties props = new Properties(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(PHOENIX_JDBC_URL, props); diff --git a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java index de21afff..13c7f679 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java @@ -178,7 +178,7 @@ public void testCoveredColumnUpdates() throws Exception { } @Test - public void testSelectAll() throws Exception { + public void testSelectAllAndAliasWithIndex() throws Exception { String query; ResultSet rs; @@ -226,6 +226,18 @@ public void testSelectAll() throws Exception { assertEquals("x",rs.getString("v1")); assertEquals("1",rs.getString("v2")); assertFalse(rs.next()); + + query = "SELECT v1 as foo FROM " + DATA_TABLE_FULL_NAME + " WHERE v2 = '1' ORDER BY foo"; + rs = conn.createStatement().executeQuery("EXPLAIN " + query); + assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " +INDEX_TABLE_FULL_NAME + " '1'\n" + + " SERVER TOP -1 ROWS SORTED BY [V1]\n" + + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); + + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("x",rs.getString(1)); + assertEquals("x",rs.getString("foo")); + assertFalse(rs.next()); } @Test From 6285059d450523f42d91e699f47dd841407d4189 Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Fri, 20 Sep 2013 11:19:58 -0700 Subject: [PATCH 060/102] Parallelizing writes to the index tables. We basically add a thread pool to which we submit write tasks for each index table in the batch. Each batch operates independently and either returns success or throws an exception. If any write throws an exception, we will need to replay that entire batch (at least in the current implementation), so we attempt to stop all pending index writes as soon as possible. Also added a CachingHTableFactory. Its a very simple caching algorithm - it just uses the JVM GC to release HTables when they are not used. This lets us save the effort of setting up and tearing down the table for each batch of index writes, potentially a huge gain over time. --- .../index/CannotReachIndexException.java | 22 +- .../hbase/index/CapturingAbortable.java | 76 +++++ .../salesforce/hbase/index/IndexWriter.java | 290 ++++++++++++++--- .../com/salesforce/hbase/index/Indexer.java | 10 +- .../index/table/CachingHTableFactory.java | 136 ++++++++ .../index/table/CoprocessorHTableFactory.java | 30 +- .../hbase/index/table/HTableFactory.java | 8 +- .../index/table/HTableInterfaceReference.java | 58 +--- .../salesforce/hbase/index/StubAbortable.java | 51 +++ .../hbase/index/TestIndexWriter.java | 293 ++++++++++++++++++ 10 files changed, 863 insertions(+), 111 deletions(-) create mode 100644 src/main/java/com/salesforce/hbase/index/CapturingAbortable.java create mode 100644 src/main/java/com/salesforce/hbase/index/table/CachingHTableFactory.java create mode 100644 src/test/java/com/salesforce/hbase/index/StubAbortable.java create mode 100644 src/test/java/com/salesforce/hbase/index/TestIndexWriter.java diff --git a/src/main/java/com/salesforce/hbase/index/CannotReachIndexException.java b/src/main/java/com/salesforce/hbase/index/CannotReachIndexException.java index 40c490d8..5097cfe2 100644 --- a/src/main/java/com/salesforce/hbase/index/CannotReachIndexException.java +++ b/src/main/java/com/salesforce/hbase/index/CannotReachIndexException.java @@ -37,9 +37,23 @@ @SuppressWarnings("serial") public class CannotReachIndexException extends Exception { + /** + * Cannot reach the index, but not sure of the table or the mutations that caused the failure + * @param msg more description of what happened + * @param cause original cause + */ + public CannotReachIndexException(String msg, Throwable cause) { + super(msg, cause); + } + + /** + * Failed to write the passed mutations to an index table for some reason. + * @param targetTableName index table to which we attempted to write + * @param mutations mutations that were attempted + * @param cause underlying reason for the failure + */ public CannotReachIndexException(String targetTableName, List mutations, Exception cause) { - super( - "Cannot reach index table " + targetTableName + " to update index for edit: " + mutations, - cause); + super("Failed to make index update:\n\t table: " + targetTableName + "\n\t edits: " + mutations + + "\n\tcause: " + cause == null ? "UNKNOWN" : cause.getMessage(), cause); } -} +} \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/CapturingAbortable.java b/src/main/java/com/salesforce/hbase/index/CapturingAbortable.java new file mode 100644 index 00000000..5493d7fa --- /dev/null +++ b/src/main/java/com/salesforce/hbase/index/CapturingAbortable.java @@ -0,0 +1,76 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index; + +import org.apache.hadoop.hbase.Abortable; + +/** + * {@link Abortable} that can rethrow the cause of the abort. + */ +public class CapturingAbortable implements Abortable { + + private Abortable delegate; + private Throwable cause; + private String why; + + public CapturingAbortable(Abortable delegate) { + this.delegate = delegate; + } + + @Override + public void abort(String why, Throwable e) { + if (delegate.isAborted()) { + return; + } + this.why = why; + this.cause = e; + delegate.abort(why, e); + + } + + @Override + public boolean isAborted() { + return delegate.isAborted(); + } + + /** + * Throw the cause of the abort, if this was aborted. If there was an exception causing + * the abort, re-throws that. Otherwise, just throws a generic {@link Exception} with the reason + * why the abort was caused. + * @throws Throwable the cause of the abort. + */ + public void throwCauseIfAborted() throws Throwable { + if (!this.isAborted()) { + return; + } + if (cause == null) { + throw new Exception(why); + } + throw cause; + } +} \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/IndexWriter.java b/src/main/java/com/salesforce/hbase/index/IndexWriter.java index 6e2477e4..f3dc3196 100644 --- a/src/main/java/com/salesforce/hbase/index/IndexWriter.java +++ b/src/main/java/com/salesforce/hbase/index/IndexWriter.java @@ -30,24 +30,41 @@ import java.io.IOException; import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.regionserver.wal.IndexedWALEdit; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.Threads; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.salesforce.hbase.index.table.CachingHTableFactory; +import com.salesforce.hbase.index.table.CoprocessorHTableFactory; import com.salesforce.hbase.index.table.HTableFactory; import com.salesforce.hbase.index.table.HTableInterfaceReference; import com.salesforce.hbase.index.util.ImmutableBytesPtr; @@ -55,27 +72,92 @@ /** * Do the actual work of writing to the index tables. Ensures that if we do fail to write to the * index table that we cleanly kill the region/server to ensure that the region's WAL gets replayed. + *

+ * We attempt to do the index updates in parallel using a backing threadpool. All threads are daemon + * threads, so it will not block the region from shutting down. */ -public class IndexWriter { +public class IndexWriter implements Stoppable { + public static String NUM_CONCURRENT_INDEX_WRITER_THREADS_CONF_KEY = "index.writer.threads.max"; private static final Log LOG = LogFactory.getLog(IndexWriter.class); + private static final int DEFAULT_CONCURRENT_INDEX_WRITER_THREADS = 10; + private static final String INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY = + "index.writer.threads.keepalivetime"; private final String sourceInfo; - private final Abortable abortable; + private final CapturingAbortable abortable; private final HTableFactory factory; + private ListeningExecutorService writerPool; + private boolean stopped; /** * @param sourceInfo log info string about where we are writing from * @param abortable to notify in the case of failure - * @param factory Factory to use when resolving the {@link HTableInterfaceReference}. If - * null, its assumed that the {@link HTableInterfaceReference} already has its - * factory set (e.g. by {@link HTableInterfaceReference#setFactory(HTableFactory)} - if - * its not already set, a {@link NullPointerException} is thrown. + * @param env Factory to use when resolving the {@link HTableInterfaceReference}. If null + * , its assumed that the {@link HTableInterfaceReference} already has its factory set + * (e.g. by {@link HTableInterfaceReference#setFactory(HTableFactory)} - if its not + * already set, a {@link NullPointerException} is thrown. */ - public IndexWriter(String sourceInfo, Abortable abortable, HTableFactory factory) { + public IndexWriter(String sourceInfo, Abortable abortable, RegionCoprocessorEnvironment env, + Configuration conf) { + this(sourceInfo, abortable, env, getDefaultExecutor(conf)); + } + + /** + * @param conf + * @return + */ + private static ThreadPoolExecutor getDefaultExecutor(Configuration conf) { + int maxThreads = + conf.getInt(NUM_CONCURRENT_INDEX_WRITER_THREADS_CONF_KEY, + DEFAULT_CONCURRENT_INDEX_WRITER_THREADS); + if (maxThreads == 0) { + maxThreads = 1; // is there a better default? + } + long keepAliveTime = conf.getLong(INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY, 60); + + // we prefer starting a new thread to queuing (the opposite of the usual ThreadPoolExecutor) + // since we are probably writing to a bunch of index tables in this case. Any pending requests + // are then queued up in an infinite (Integer.MAX_VALUE) queue. However, we allow core threads + // to timeout, to we tune up/down for bursty situations. We could be a bit smarter and more + // closely manage the core-thread pool size to handle the bursty traffic (so we can always keep + // some core threads on hand, rather than starting from scratch each time), but that would take + // even more time. If we shutdown the pool, but are still putting new tasks, we can just do the + // usual policy and throw a RejectedExecutionException because we are shutting down anyways and + // the worst thing is that this gets unloaded. + ThreadPoolExecutor pool = + new ThreadPoolExecutor(maxThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS, + new LinkedBlockingQueue(), Threads.newDaemonThreadFactory("index-writer-")); + pool.allowCoreThreadTimeOut(true); + return pool; + } + + public IndexWriter(String sourceInfo, Abortable abortable, CoprocessorEnvironment env, + ExecutorService pool) { + this(sourceInfo, abortable, pool, getDefaultDelegateHTableFactory(env)); + } + + private static HTableFactory getDefaultDelegateHTableFactory(CoprocessorEnvironment env) { + // create a simple delegate factory, setup the way we need + Configuration conf = env.getConfiguration(); + // only have one thread per table - all the writes are already batched per table. + + conf.setInt("hbase.htable.threads.max", 1); + return new CoprocessorHTableFactory(env); + } + + /** + * Internal constructor - Exposed for testing! + * @param sourceInfo + * @param abortable + * @param pool + * @param delegate + */ + IndexWriter(String sourceInfo, Abortable abortable, ExecutorService pool, HTableFactory delegate) { this.sourceInfo = sourceInfo; - this.abortable = abortable; - this.factory = factory; + this.abortable = new CapturingAbortable(abortable); + this.writerPool = MoreExecutors.listeningDecorator(pool); + this.factory = new CachingHTableFactory(delegate); } /** @@ -94,11 +176,18 @@ public void writeAndKillYourselfOnFailure(Collection> ind killYourself(e); } } - + /** - * Write the mutations to their respective table using the provided factory. + * Write the mutations to their respective table. *

- * This method is not thread-safe and if accessed in a non-serial manner could leak HTables. + * This method is blocking and could potentially cause the writer to block for a long time as we + * write the index updates. We only return when either: + *

    + *
  1. All index writes have returned, OR
    2. + *
  2. Any single index write has failed
    3. + *
+ * We attempt to quickly determine if any write has failed and not write to the remaining indexes + * to ensure a timely recovery of the failed index writes. * @param indexUpdates Updates to write * @throws CannotReachIndexException if we cannot successfully write a single index entry. We stop * immediately on the first failed index write, rather than attempting all writes. @@ -106,48 +195,132 @@ public void writeAndKillYourselfOnFailure(Collection> ind public void write(Collection> indexUpdates) throws CannotReachIndexException { // convert the strings to htableinterfaces to which we can talk and group by TABLE - Multimap toWrite = - resolveTableReferences(factory, indexUpdates); + Multimap toWrite = resolveTableReferences(factory, + indexUpdates); + + /* + * This bit here is a little odd, so let's explain what's going on. Basically, we want to do the + * writes in parallel to each index table, so each table gets its own task and is submitted to + * the pool. Where it gets tricky is that we want to block the calling thread until one of two + * things happens: (1) all index tables get successfully updated, or (2) any one of the index + * table writes fail; in either case, we should return as quickly as possible. We get a little + * more complicated in that if we do get a single failure, but any of the index writes hasn't + * been started yet (its been queued up, but not submitted to a thread) we want to that task to + * fail immediately as we know that write is a waste and will need to be replayed anyways. + */ - // write each mutation, as a part of a batch, to its respective table - List mutations; - Set tables = new HashSet(); - for (Entry> entry : toWrite.asMap().entrySet()) { + Set>> entries = toWrite.asMap().entrySet(); + CompletionService ops = new ExecutorCompletionService(this.writerPool); + for (Entry> entry : entries) { // get the mutations for each table. We leak the implementation here a little bit to save // doing a complete copy over of all the index update for each table. - mutations = (List) entry.getValue(); - if (LOG.isDebugEnabled()) { - LOG.debug("Writing index update:" + mutations + " to table: " - + entry.getKey().getTableName()); + final List mutations = (List) entry.getValue(); + final HTableInterfaceReference tableReference = entry.getKey(); + // early exit - no need to submit new tasks if we are shutting down + if (this.stopped || this.abortable.isAborted()) { + break; } - try { - HTableInterface table; - if (factory == null) { - table = entry.getKey().getTable(); - } else { - table = entry.getKey().getTable(factory); + /* + * Write a batch of index updates to an index table. This operation stops (is cancelable) via + * two mechanisms: (1) setting aborted or stopped on the IndexWriter or, (2) interrupting the + * running thread. The former will only work if we are not in the midst of writing the current + * batch to the table, though we do check these status variables before starting and before + * writing the batch. The latter usage, interrupting the thread, will work in the previous + * situations as was at some points while writing the batch, depending on the underlying + * writer implementation (HTableInterface#batch is blocking, but doesn't elaborate when is + * supports an interrupt). + */ + ops.submit(new Callable() { + + /** + * Do the actual write to the primary table. We don't need to worry about closing the table + * because that is handled the {@link CachingHTableFactory}. + */ + @Override + public Void call() throws Exception { + // this may have been queued, so another task infront of us may have failed, so we should + // early exit, if that's the case + throwFailureIfDone(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Writing index update:" + mutations + " to table: " + tableReference); + } + try { + HTableInterface table = factory.getTable(tableReference.get()); + throwFailureIfDone(); + table.batch(mutations); + } catch (CannotReachIndexException e) { + throw e; + } catch (IOException e) { + throw new CannotReachIndexException(tableReference.toString(), mutations, e); + } catch (InterruptedException e) { + // reset the interrupt status on the thread + Thread.currentThread().interrupt(); + throw new CannotReachIndexException(tableReference.toString(), mutations, e); + } + return null; } - table.batch(mutations); - tables.add(table); - } catch (IOException e) { - throw new CannotReachIndexException(entry.getKey().getTableName(), mutations, e); - } catch (InterruptedException e) { - throw new CannotReachIndexException(entry.getKey().getTableName(), mutations, e); - } + + private void throwFailureIfDone() throws CannotReachIndexException { + if (stopped || abortable.isAborted() || Thread.currentThread().isInterrupted()) { + throw new CannotReachIndexException( + "Pool closed, not attempting to write to the index!", null); + } + + } + }); } - // go through each reference and close the connection - // we can't do this earlier as we may reuse table references between different index entries, - // which would prematurely close a table before we could write the later update - for (HTableInterface table : tables) { + + boolean interrupted = false; + // we can use a simple counter here because its ever only modified by the waiting thread + int completedWrites = 0; + /* + * wait for all index writes to complete, or there to be a failure. We could be faster here in + * terms of watching for a failed index write. Right now, we need to wade through any successful + * attempts that happen to finish before we get to the failed update. For right now, that's fine + * as we don't really spend a lot time getting through the successes and a slight delay on the + * abort really isn't the end of the world. We could be smarter and use a Guava ListenableFuture + * to handle a callback off the future that updates the abort status, but for now we don't need + * the extra complexity. + */ + while (!this.abortable.isAborted() && !this.isStopped() && completedWrites < entries.size()) { try { - table.close(); - } catch (IOException e) { - LOG.error("Failed to close connection to table:" + Bytes.toString(table.getTableName()), e); + Future status = ops.take(); + try { + // we don't care what the status is - success is binary, so no error == success + status.get(); + completedWrites++; + } catch (CancellationException e) { + // if we get a cancellation, we already failed for some other reason, so we can ignore it + LOG.debug("Found canceled index write - ignoring!"); + } catch (ExecutionException e) { + LOG.error("Found a failed index update!"); + abortable.abort("Failed ot writer to an index table!", e.getCause()); + break; + } + } catch (InterruptedException e) { + LOG.info("Index writer interrupted, continuing if not aborted or stopped."); + // reset the interrupt status so we can wait out that latch + interrupted = true; } } + // reset the interrupt status after we are done + if (interrupted) { + Thread.currentThread().interrupt(); + } - LOG.info("Done writing all index updates"); + // propagate the failure up to the caller + try { + this.abortable.throwCauseIfAborted(); + } catch (CannotReachIndexException e) { + throw e; + } catch (Throwable e) { + throw new CannotReachIndexException("Got an abort notification while writing to the index!", + e); + } + + LOG.info("Done writing all index updates!"); } /** @@ -155,6 +328,9 @@ public void write(Collection> indexUpdates) * @param info region from which we are attempting to write the log */ private void killYourself(Throwable cause) { + // cleanup resources + this.stop("Killing ourselves because of an error:" + cause); + // notify the regionserver of the failure String msg = "Could not update the index table, killing server region from: " + this.sourceInfo; LOG.error(msg); try { @@ -174,8 +350,8 @@ private void killYourself(Throwable cause) { */ public static Multimap resolveTableReferences( HTableFactory factory, Collection> indexUpdates) { - Multimap updates = - ArrayListMultimap. create(); + Multimap updates = ArrayListMultimap + . create(); // simple map to make lookups easy while we build the map of tables to create Map tables = new HashMap(updates.size()); @@ -184,7 +360,7 @@ public static Multimap resolveTableReference ImmutableBytesPtr ptr = new ImmutableBytesPtr(tableName); HTableInterfaceReference table = tables.get(ptr); if (table == null) { - table = new HTableInterfaceReference(ptr, factory); + table = new HTableInterfaceReference(ptr); tables.put(ptr, table); } updates.put(table, entry.getFirst()); @@ -192,4 +368,20 @@ public static Multimap resolveTableReference return updates; } -} + + @Override + public void stop(String why) { + if (this.stopped) { + return; + } + this.stopped = true; + LOG.debug("Stopping because " + why); + this.writerPool.shutdownNow(); + this.factory.shutdown(); + } + + @Override + public boolean isStopped() { + return this.stopped; + } +} \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/Indexer.java b/src/main/java/com/salesforce/hbase/index/Indexer.java index f9310794..06ed3329 100644 --- a/src/main/java/com/salesforce/hbase/index/Indexer.java +++ b/src/main/java/com/salesforce/hbase/index/Indexer.java @@ -86,8 +86,6 @@ public class Indexer extends BaseRegionObserver { /** WAL on this server */ private HLog log; protected IndexWriter writer; - protected HTableFactory factory; - protected IndexBuilder builder; /** Configuration key for the {@link IndexBuilder} to use */ @@ -107,7 +105,6 @@ public class Indexer extends BaseRegionObserver { @Override public void start(CoprocessorEnvironment e) throws IOException { - this.factory = new CoprocessorHTableFactory(e); final RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) e; @@ -143,7 +140,12 @@ public void start(CoprocessorEnvironment e) throws IOException { // and setup the actual index writer this.writer = new IndexWriter("Region: " + env.getRegion().getRegionNameAsString(), - env.getRegionServerServices(), factory); + env.getRegionServerServices(), env, conf); + } + + @Override + public void stop(CoprocessorEnvironment e) throws IOException { + this.writer.stop("Indexer is being stopped"); } @Override diff --git a/src/main/java/com/salesforce/hbase/index/table/CachingHTableFactory.java b/src/main/java/com/salesforce/hbase/index/table/CachingHTableFactory.java new file mode 100644 index 00000000..f20520bb --- /dev/null +++ b/src/main/java/com/salesforce/hbase/index/table/CachingHTableFactory.java @@ -0,0 +1,136 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index.table; + +import java.io.IOException; +import java.lang.ref.ReferenceQueue; +import java.lang.ref.SoftReference; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.util.Bytes; + +import com.salesforce.hbase.index.util.ImmutableBytesPtr; + +/** + * A simple cache that just uses usual GC mechanisms to cleanup unused {@link HTableInterface}s. + * When requesting an {@link HTableInterface} via {@link #getTable(byte[])}, you may get the same + * table as last time, or it may be a new table. + *

+ * You should not call {@link HTableInterface#close()} that is handled when the table goes + * out of scope. Along the same lines, you must ensure to not keep a reference to the table for + * longer than necessary - this leak will ensure that the table never gets closed. + */ +public class CachingHTableFactory implements HTableFactory { + + private static final Log LOG = LogFactory.getLog(CachingHTableFactory.class); + private HTableFactory delegate; + + Map> openTables = + new HashMap>(); + + private ReferenceQueue referenceQueue; + + private Thread cleanupThread; + + public CachingHTableFactory(HTableFactory tableFactory) { + this.delegate = tableFactory; + this.referenceQueue = new ReferenceQueue(); + this.cleanupThread = new Thread(new TableCleaner(referenceQueue), "cached-table-cleanup"); + cleanupThread.setDaemon(true); + cleanupThread.start(); + } + + @Override + public HTableInterface getTable(ImmutableBytesPtr tablename) throws IOException { + ImmutableBytesPtr tableBytes = new ImmutableBytesPtr(tablename); + synchronized (openTables) { + SoftReference ref = openTables.get(tableBytes); + // the reference may be null, in which case this is a new table, or the underlying HTable may + // have been GC'ed. + @SuppressWarnings("resource") + HTableInterface table = ref == null ? null : ref.get(); + if (table == null) { + table = delegate.getTable(tablename); + openTables.put(tableBytes, new SoftReference(table, referenceQueue)); + } + return table; + } + } + + @Override + public void shutdown() { + this.cleanupThread.interrupt(); + this.delegate.shutdown(); + } + + /** + * Cleaner to ensure that any tables that are GC'ed are also closed. + */ + private class TableCleaner implements Runnable { + + private ReferenceQueue queue; + + public TableCleaner(ReferenceQueue referenceQueue) { + this.queue = referenceQueue; + } + + @Override + public void run() { + try { + HTableInterface table = this.queue.remove().get(); + if (table != null) { + try { + table.close(); + } catch (IOException e) { + LOG.error( + "Failed to correctly close htable, ignoring it further since it is being GC'ed", e); + } + } + } catch (InterruptedException e) { + LOG.info("Recieved an interrupt - assuming system is going down. Closing all remaining HTables and quitting!"); + for (SoftReference ref : openTables.values()) { + HTableInterface table = ref.get(); + if (table != null) { + try { + LOG.info("Closing connection to index table: " + Bytes.toString(table.getTableName())); + table.close(); + } catch (IOException ioe) { + LOG.error( + "Failed to correctly close htable on shutdown! Ignoring and closing remaining tables", + ioe); + } + } + } + } + } + } +} \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/table/CoprocessorHTableFactory.java b/src/main/java/com/salesforce/hbase/index/table/CoprocessorHTableFactory.java index 8ee049dc..b017ac8b 100644 --- a/src/main/java/com/salesforce/hbase/index/table/CoprocessorHTableFactory.java +++ b/src/main/java/com/salesforce/hbase/index/table/CoprocessorHTableFactory.java @@ -7,6 +7,8 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.HTableInterface; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; + public class CoprocessorHTableFactory implements HTableFactory { private CoprocessorEnvironment e; @@ -16,7 +18,7 @@ public CoprocessorHTableFactory(CoprocessorEnvironment e) { } @Override - public HTableInterface getTable(byte[] tablename) throws IOException { + public HTableInterface getTable(ImmutableBytesPtr tablename) throws IOException { Configuration conf = e.getConfiguration(); // make sure writers fail fast conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3); @@ -26,6 +28,30 @@ public HTableInterface getTable(byte[] tablename) throws IOException { conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 30000); conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 5000); - return this.e.getTable(tablename); + return this.e.getTable(copyOrBytesIfExact(tablename)); + } + + /** + * There are possible outcomes: a copy of the section of bytes that we care about, or the exact + * byte array, if the {@link ImmutableBytesPtr} doesn't map 'into' the byte array. This saves us + * doing an extra byte copy for each table name. + *

+ * Either way, you should not modify the returned bytes - it should be assumed that they are + * backing the {@link ImmutableBytesPtr}. + * @param bytes to introspect + * @return a byte[] from the {@link ImmutableBytesPtr} + */ + private byte[] copyOrBytesIfExact(ImmutableBytesPtr bytes) { + if (bytes.getOffset() == 0) { + if (bytes.getLength() == bytes.get().length) { + return bytes.get(); + } + } + return bytes.copyBytes(); + } + + @Override + public void shutdown() { + // noop } } \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/table/HTableFactory.java b/src/main/java/com/salesforce/hbase/index/table/HTableFactory.java index e00e5d30..6dca1da0 100644 --- a/src/main/java/com/salesforce/hbase/index/table/HTableFactory.java +++ b/src/main/java/com/salesforce/hbase/index/table/HTableFactory.java @@ -4,7 +4,11 @@ import org.apache.hadoop.hbase.client.HTableInterface; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; + public interface HTableFactory { - public HTableInterface getTable(byte [] tablename) throws IOException; -} + public HTableInterface getTable(ImmutableBytesPtr tablename) throws IOException; + + public void shutdown(); +} \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/table/HTableInterfaceReference.java b/src/main/java/com/salesforce/hbase/index/table/HTableInterfaceReference.java index 0c3472ca..f6ad2029 100644 --- a/src/main/java/com/salesforce/hbase/index/table/HTableInterfaceReference.java +++ b/src/main/java/com/salesforce/hbase/index/table/HTableInterfaceReference.java @@ -1,12 +1,8 @@ package com.salesforce.hbase.index.table; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.client.HTableInterface; -import org.apache.hadoop.io.Writable; +import org.apache.hadoop.hbase.util.Bytes; import com.salesforce.hbase.index.util.ImmutableBytesPtr; @@ -26,64 +22,23 @@ * multi-threaded usage must employ external locking to ensure that multiple {@link HTableInterface} * s are not resolved. */ -public class HTableInterfaceReference implements Writable { +public class HTableInterfaceReference { private ImmutableBytesPtr tableName; - private HTableInterface table; - private HTableFactory factory; - /** - * For use with {@link #readFields(DataInput)}. A {@link HTableFactory} must be passed either to - * {@link #setFactory(HTableFactory)} before resolving an HTableInterface or - * {@link #getTable(HTableFactory)} when resolving an {@link HTableInterface} - */ - public HTableInterfaceReference() { - } public HTableInterfaceReference(ImmutableBytesPtr tableName) { this.tableName = tableName; } - public HTableInterfaceReference(ImmutableBytesPtr tableName, HTableFactory factory) { - this(tableName); - this.factory = factory; - } - - public void setFactory(HTableFactory e) { - this.factory = e; - } - - public HTableInterface getTable(HTableFactory e) throws IOException { - if (this.table == null) { - this.table = e.getTable(this.tableName.copyBytesIfNecessary()); - } - return this.table; - } - - /** - * @return get the referenced table, if one has been stored - * @throws IOException if we are creating a new table (first instance of request) and it cannot be - * reached - */ - public HTableInterface getTable() throws IOException { - return this.getTable(this.factory); + public ImmutableBytesPtr get() { + return this.tableName; } public String getTableName() { return this.tableName.toString(); } - @Override - public void readFields(DataInput in) throws IOException { - this.tableName = new ImmutableBytesPtr(); - this.tableName.readFields(in); - } - - @Override - public void write(DataOutput out) throws IOException { - this.tableName.write(out); - } - @Override public int hashCode() { return tableName.hashCode(); @@ -98,5 +53,8 @@ public boolean equals(Object obj) { return tableName.equals(other.tableName); } - + @Override + public String toString() { + return Bytes.toString(this.tableName.get()); + } } \ No newline at end of file diff --git a/src/test/java/com/salesforce/hbase/index/StubAbortable.java b/src/test/java/com/salesforce/hbase/index/StubAbortable.java new file mode 100644 index 00000000..3551c8a9 --- /dev/null +++ b/src/test/java/com/salesforce/hbase/index/StubAbortable.java @@ -0,0 +1,51 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Abortable; + +/** + * TEst helper to stub out an {@link Abortable} when needed. + */ +public class StubAbortable implements Abortable { + private static final Log LOG = LogFactory.getLog(StubAbortable.class); + private boolean abort; + + @Override + public void abort(String reason, Throwable e) { + LOG.info("Aborting: " + reason, e); + abort = true; + } + + @Override + public boolean isAborted() { + return abort; + } +} diff --git a/src/test/java/com/salesforce/hbase/index/TestIndexWriter.java b/src/test/java/com/salesforce/hbase/index/TestIndexWriter.java new file mode 100644 index 00000000..faad8f36 --- /dev/null +++ b/src/test/java/com/salesforce/hbase/index/TestIndexWriter.java @@ -0,0 +1,293 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import com.salesforce.hbase.index.table.HTableFactory; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; + +public class TestIndexWriter { + private static final Log LOG = LogFactory.getLog(TestIndexWriter.class); + @Rule + public TableName testName = new TableName(); + + /** + * Simple table factory that just looks up the tables based on name. Useful for mocking up + * {@link HTableInterface}s without having to mock up the factory too. + */ + private class FakeTableFactory implements HTableFactory { + + boolean shutdown = false; + private Map tables; + + public FakeTableFactory(Map tables) { + this.tables = tables; + } + + @Override + public HTableInterface getTable(ImmutableBytesPtr tablename) throws IOException { + return this.tables.get(tablename); + } + + @Override + public void shutdown() { + shutdown = true; + } + } + + private final byte[] row = Bytes.toBytes("row"); + + /** + * Test that we correctly shutdown/cleanup all the resources the writer creates + * @throws Exception on failure + */ + @Test + public void correctlyCleanupResources() throws Exception { + Abortable abort = new StubAbortable(); + ExecutorService exec = Executors.newFixedThreadPool(1); + FakeTableFactory factory = new FakeTableFactory( + Collections. emptyMap()); + + // create a simple writer + IndexWriter writer = new IndexWriter(this.testName.getTableNameString(), abort, exec, factory); + writer.stop(this.testName.getTableNameString() + " finished"); + assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown); + assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown()); + } + + /** + * With the move to using a pool of threads to write, we need to ensure that we still block until + * all index writes for a mutation/batch are completed. + * @throws Exception on failure + */ + @Test + public void testSynchronouslyCompletesAllWrites() throws Exception { + LOG.info("Starting " + testName.getTableNameString()); + LOG.info("Current thread is interrupted: " + Thread.interrupted()); + // LOG.info("Current thread is interrupted: " + Thread.interrupted()); + Abortable abort = new StubAbortable(); + ExecutorService exec = Executors.newFixedThreadPool(1); + Map tables = new HashMap(); + FakeTableFactory factory = new FakeTableFactory(tables); + + byte[] tableName = this.testName.getTableName(); + Put m = new Put(row); + m.add(Bytes.toBytes("family"), Bytes.toBytes("qual"), null); + Collection> indexUpdates = Arrays.asList(new Pair(m, + tableName)); + + HTableInterface table = Mockito.mock(HTableInterface.class); + final boolean[] completed = new boolean[] { false }; + Mockito.when(table.batch(Mockito.anyList())).thenAnswer(new Answer() { + + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + // just keep track that it was called + completed[0] = true; + return null; + } + }); + Mockito.when(table.getTableName()).thenReturn(testName.getTableName()); + // add the table to the set of tables, so its returned to the writer + tables.put(new ImmutableBytesPtr(tableName), table); + + IndexWriter writer = new IndexWriter(this.testName.getTableNameString(), abort, exec, factory); + writer.write(indexUpdates); + assertTrue("Writer returned before the table batch completed! Likely a race condition tripped", + completed[0]); + writer.stop(this.testName.getTableNameString() + " finished"); + assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown); + assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown()); + } + + /** + * Index updates can potentially be queued up if there aren't enough writer threads. If a running + * index write fails, then we should early exit the pending indexupdate, when it comes up (if the + * pool isn't already shutdown). + *

+ * This test is a little bit racey - we could actually have the failure of the first task before + * the third task is even submitted. However, we should never see the third task attempt to make + * the batch write, so we should never see a failure here. + * @throws Exception on failure + */ + // @Ignore + @Test + public void testFailureOnRunningUpdateAbortsPending() throws Exception { + Abortable abort = new StubAbortable(); + // single thread factory so the older request gets queued + ExecutorService exec = Executors.newFixedThreadPool(1); + Map tables = new HashMap(); + FakeTableFactory factory = new FakeTableFactory(tables); + + // updates to two different tables + byte[] tableName = Bytes.add(this.testName.getTableName(), new byte[] { 1, 2, 3, 4 }); + Put m = new Put(row); + m.add(Bytes.toBytes("family"), Bytes.toBytes("qual"), null); + byte[] tableName2 = this.testName.getTableName();// this will sort after the first tablename + List> indexUpdates = new ArrayList>(); + indexUpdates.add(new Pair(m, tableName)); + indexUpdates.add(new Pair(m, tableName2)); + indexUpdates.add(new Pair(m, tableName2)); + + // first table will fail + HTableInterface table = Mockito.mock(HTableInterface.class); + Mockito.when(table.batch(Mockito.anyList())).thenThrow( + new IOException("Intentional IOException for failed first write.")); + Mockito.when(table.getTableName()).thenReturn(tableName); + + // second table just blocks to make sure that the abort propagates to the third task + final CountDownLatch waitOnAbortedLatch = new CountDownLatch(1); + final boolean[] failed = new boolean[] { false }; + HTableInterface table2 = Mockito.mock(HTableInterface.class); + Mockito.when(table2.getTableName()).thenReturn(tableName2); + Mockito.when(table2.batch(Mockito.anyList())).thenAnswer(new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + waitOnAbortedLatch.await(); + return null; + } + }).thenAnswer(new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + failed[0] = true; + throw new RuntimeException( + "Unexpected exception - second index table shouldn't have been written to"); + } + }); + + // add the tables to the set of tables, so its returned to the writer + tables.put(new ImmutableBytesPtr(tableName), table); + tables.put(new ImmutableBytesPtr(tableName2), table2); + + IndexWriter writer = new IndexWriter(this.testName.getTableNameString(), abort, exec, factory); + try { + writer.write(indexUpdates); + fail("Should not have successfully completed all index writes"); + } catch (CannotReachIndexException e) { + LOG.info("Correctly got a failure to reach the index", e); + // should have correctly gotten the correct abort, so let the next task execute + waitOnAbortedLatch.countDown(); + } + assertFalse( + "Third set of index writes never have been attempted - should have seen the abort before done!", + failed[0]); + writer.stop(this.testName.getTableNameString() + " finished"); + assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown); + assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown()); + } + + /** + * Test that if we get an interruption to to the thread while doing a batch (e.g. via shutdown), + * that we correctly end the task + * @throws Exception on failure + */ + @Test + public void testShutdownInterruptsAsExpected() throws Exception { + + Abortable abort = new StubAbortable(); + // single thread factory so the older request gets queued + ExecutorService exec = Executors.newFixedThreadPool(1); + Map tables = new HashMap(); + FakeTableFactory factory = new FakeTableFactory(tables); + + byte[] tableName = this.testName.getTableName(); + HTableInterface table = Mockito.mock(HTableInterface.class); + Mockito.when(table.getTableName()).thenReturn(tableName); + final CountDownLatch writeStartedLatch = new CountDownLatch(1); + final CountDownLatch waitOnAbortedLatch = new CountDownLatch(1); + Mockito.when(table.batch(Mockito.anyList())).thenAnswer(new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + writeStartedLatch.countDown(); + // when we interrupt the thread for shutdown, we should see this throw an interrupt too + waitOnAbortedLatch.await(); + return null; + } + }); + // add the tables to the set of tables, so its returned to the writer + tables.put(new ImmutableBytesPtr(tableName), table); + + // update a single table + Put m = new Put(row); + m.add(Bytes.toBytes("family"), Bytes.toBytes("qual"), null); + final List> indexUpdates = new ArrayList>(); + indexUpdates.add(new Pair(m, tableName)); + + // setup the writer + final IndexWriter writer = new IndexWriter(this.testName.getTableNameString(), abort, exec, + factory); + + final boolean[] failedWrite = new boolean[] { false }; + Thread primaryWriter = new Thread() { + + @Override + public void run() { + try { + writer.write(indexUpdates); + } catch (CannotReachIndexException e) { + failedWrite[0] = true; + } + } + }; + primaryWriter.start(); + // wait for the write to start before intentionally shutdown the pool + writeStartedLatch.await(); + writer.stop("Shutting down writer for test " + this.testName.getTableNameString()); + primaryWriter.join(); + assertTrue("Writer should have failed because of the stop we issued", failedWrite[0]); + } +} \ No newline at end of file From a45f2a0613a426722f7b41cfa9efefec2a6ba490 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sun, 22 Sep 2013 01:50:24 -0700 Subject: [PATCH 061/102] Sort on client instead of server for grouped aggregation to fix issue #417 --- .../com/salesforce/hbase/index/Indexer.java | 2 - .../phoenix/compile/DeleteCompiler.java | 26 +++++-- .../phoenix/compile/PostDDLCompiler.java | 3 +- .../phoenix/compile/QueryCompiler.java | 3 +- .../phoenix/compile/UpsertCompiler.java | 6 +- .../GroupedAggregateRegionObserver.java | 39 +++++++--- .../phoenix/execute/AggregatePlan.java | 75 ++++++++++++++++++- .../salesforce/phoenix/execute/ScanPlan.java | 25 +++++-- .../phoenix/expression/RowKeyExpression.java | 25 +++++++ .../iterate/OrderedResultIterator.java | 33 ++++++-- .../hbase/index/TestIndexWriter.java | 5 +- .../phoenix/end2end/QueryExecTest.java | 67 +++++++++++++++++ 12 files changed, 265 insertions(+), 44 deletions(-) create mode 100644 src/main/java/com/salesforce/phoenix/expression/RowKeyExpression.java diff --git a/src/main/java/com/salesforce/hbase/index/Indexer.java b/src/main/java/com/salesforce/hbase/index/Indexer.java index 06ed3329..cf80649d 100644 --- a/src/main/java/com/salesforce/hbase/index/Indexer.java +++ b/src/main/java/com/salesforce/hbase/index/Indexer.java @@ -63,8 +63,6 @@ import org.apache.hadoop.hbase.util.Pair; import com.salesforce.hbase.index.builder.IndexBuilder; -import com.salesforce.hbase.index.table.CoprocessorHTableFactory; -import com.salesforce.hbase.index.table.HTableFactory; import com.salesforce.hbase.index.wal.IndexedKeyValue; /** diff --git a/src/main/java/com/salesforce/phoenix/compile/DeleteCompiler.java b/src/main/java/com/salesforce/phoenix/compile/DeleteCompiler.java index 7812aef3..069e3205 100644 --- a/src/main/java/com/salesforce/phoenix/compile/DeleteCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/DeleteCompiler.java @@ -29,7 +29,9 @@ import java.sql.ParameterMetaData; import java.sql.SQLException; -import java.util.*; +import java.util.Collections; +import java.util.List; +import java.util.Map; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Scan; @@ -41,17 +43,27 @@ import com.salesforce.phoenix.compile.GroupByCompiler.GroupBy; import com.salesforce.phoenix.compile.OrderByCompiler.OrderBy; import com.salesforce.phoenix.coprocessor.UngroupedAggregateRegionObserver; -import com.salesforce.phoenix.execute.*; +import com.salesforce.phoenix.execute.AggregatePlan; +import com.salesforce.phoenix.execute.MutationState; +import com.salesforce.phoenix.execute.ScanPlan; import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.expression.LiteralExpression; import com.salesforce.phoenix.iterate.ParallelIterators.ParallelIteratorFactory; -import com.salesforce.phoenix.iterate.*; +import com.salesforce.phoenix.iterate.ResultIterator; import com.salesforce.phoenix.iterate.SpoolingResultIterator.SpoolingResultIteratorFactory; import com.salesforce.phoenix.jdbc.PhoenixConnection; -import com.salesforce.phoenix.parse.*; -import com.salesforce.phoenix.query.*; +import com.salesforce.phoenix.parse.DeleteStatement; +import com.salesforce.phoenix.parse.SelectStatement; +import com.salesforce.phoenix.query.ConnectionQueryServices; +import com.salesforce.phoenix.query.QueryConstants; +import com.salesforce.phoenix.query.QueryServices; +import com.salesforce.phoenix.query.QueryServicesOptions; import com.salesforce.phoenix.query.Scanner; -import com.salesforce.phoenix.schema.*; +import com.salesforce.phoenix.schema.PColumn; +import com.salesforce.phoenix.schema.PDataType; +import com.salesforce.phoenix.schema.PTableType; +import com.salesforce.phoenix.schema.ReadOnlyTableException; +import com.salesforce.phoenix.schema.TableRef; import com.salesforce.phoenix.schema.tuple.Tuple; public class DeleteCompiler { @@ -156,7 +168,7 @@ public PhoenixConnection getConnection() { // Ignoring ORDER BY, since with auto commit on and no limit makes no difference SelectStatement select = SelectStatement.create(SelectStatement.COUNT_ONE, statement.getHint()); final RowProjector projector = ProjectionCompiler.compile(context, select, GroupBy.EMPTY_GROUP_BY); - final QueryPlan plan = new AggregatePlan(context, select, tableRef, projector, null, OrderBy.EMPTY_ORDER_BY, new SpoolingResultIteratorFactory(services), GroupBy.EMPTY_GROUP_BY, null); + final QueryPlan plan = new AggregatePlan(context, select, tableRef, projector, null, OrderBy.EMPTY_ORDER_BY, null, GroupBy.EMPTY_GROUP_BY, null); return new MutationPlan() { @Override diff --git a/src/main/java/com/salesforce/phoenix/compile/PostDDLCompiler.java b/src/main/java/com/salesforce/phoenix/compile/PostDDLCompiler.java index ce04bf50..d9379e50 100644 --- a/src/main/java/com/salesforce/phoenix/compile/PostDDLCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/PostDDLCompiler.java @@ -42,7 +42,6 @@ import com.salesforce.phoenix.execute.AggregatePlan; import com.salesforce.phoenix.execute.MutationState; import com.salesforce.phoenix.iterate.ResultIterator; -import com.salesforce.phoenix.iterate.SpoolingResultIterator.SpoolingResultIteratorFactory; import com.salesforce.phoenix.jdbc.PhoenixConnection; import com.salesforce.phoenix.jdbc.PhoenixParameterMetaData; import com.salesforce.phoenix.parse.SelectStatement; @@ -157,7 +156,7 @@ public ColumnRef resolveColumn(String schemaName, String tableName, String colNa } projector = new RowProjector(projector,false); } - QueryPlan plan = new AggregatePlan(context, SelectStatement.COUNT_ONE, tableRef, projector, null, OrderBy.EMPTY_ORDER_BY, new SpoolingResultIteratorFactory(connection.getQueryServices()), GroupBy.EMPTY_GROUP_BY, null); + QueryPlan plan = new AggregatePlan(context, SelectStatement.COUNT_ONE, tableRef, projector, null, OrderBy.EMPTY_ORDER_BY, null, GroupBy.EMPTY_GROUP_BY, null); Scanner scanner = plan.getScanner(); ResultIterator iterator = scanner.iterator(); try { diff --git a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java index e28f7935..68dd57ca 100644 --- a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java @@ -40,7 +40,6 @@ import com.salesforce.phoenix.execute.ScanPlan; import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.iterate.ParallelIterators.ParallelIteratorFactory; -import com.salesforce.phoenix.iterate.SpoolingResultIterator.SpoolingResultIteratorFactory; import com.salesforce.phoenix.jdbc.PhoenixConnection; import com.salesforce.phoenix.jdbc.PhoenixDatabaseMetaData; import com.salesforce.phoenix.parse.SelectStatement; @@ -81,7 +80,7 @@ public QueryCompiler(PhoenixConnection connection, int maxRows) { } public QueryCompiler(PhoenixConnection connection, int maxRows, Scan scan) { - this(connection, maxRows, scan, null, new SpoolingResultIteratorFactory(connection.getQueryServices())); + this(connection, maxRows, scan, null, null); } public QueryCompiler(PhoenixConnection connection, int maxRows, PColumn[] targetDatums, ParallelIteratorFactory parallelIteratorFactory) { diff --git a/src/main/java/com/salesforce/phoenix/compile/UpsertCompiler.java b/src/main/java/com/salesforce/phoenix/compile/UpsertCompiler.java index 80fe09f0..68d0ba18 100644 --- a/src/main/java/com/salesforce/phoenix/compile/UpsertCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/UpsertCompiler.java @@ -54,8 +54,6 @@ import com.salesforce.phoenix.expression.LiteralExpression; import com.salesforce.phoenix.iterate.ParallelIterators.ParallelIteratorFactory; import com.salesforce.phoenix.iterate.ResultIterator; -import com.salesforce.phoenix.iterate.SpoolingResultIterator; -import com.salesforce.phoenix.iterate.SpoolingResultIterator.SpoolingResultIteratorFactory; import com.salesforce.phoenix.jdbc.PhoenixConnection; import com.salesforce.phoenix.jdbc.PhoenixResultSet; import com.salesforce.phoenix.jdbc.PhoenixStatement; @@ -282,7 +280,7 @@ public MutationPlan compile(UpsertStatement upsert, List binds) throws S ParallelIteratorFactory parallelIteratorFactory; // TODO: once MutationState is thread safe, then when auto commit is off, we can still run in parallel if (select.isAggregate() || select.isDistinct() || select.getLimit() != null) { - parallelIteratorFactory = new SpoolingResultIterator.SpoolingResultIteratorFactory(services); + parallelIteratorFactory = null; } else { // We can pipeline the upsert select instead of spooling everything to disk first, // if we don't have any post processing that's required. @@ -402,7 +400,7 @@ public MutationPlan compile(UpsertStatement upsert, List binds) throws S scan.setAttribute(UngroupedAggregateRegionObserver.UPSERT_SELECT_TABLE, UngroupedAggregateRegionObserver.serialize(projectedTable)); scan.setAttribute(UngroupedAggregateRegionObserver.UPSERT_SELECT_EXPRS, UngroupedAggregateRegionObserver.serialize(projectedExpressions)); // Ignore order by - it has no impact - final QueryPlan aggPlan = new AggregatePlan(context, select, tableRef, projector, null, OrderBy.EMPTY_ORDER_BY, new SpoolingResultIteratorFactory(services), GroupBy.EMPTY_GROUP_BY, null); + final QueryPlan aggPlan = new AggregatePlan(context, select, tableRef, projector, null, OrderBy.EMPTY_ORDER_BY, null, GroupBy.EMPTY_GROUP_BY, null); return new MutationPlan() { @Override diff --git a/src/main/java/com/salesforce/phoenix/coprocessor/GroupedAggregateRegionObserver.java b/src/main/java/com/salesforce/phoenix/coprocessor/GroupedAggregateRegionObserver.java index b70516d7..32640b58 100644 --- a/src/main/java/com/salesforce/phoenix/coprocessor/GroupedAggregateRegionObserver.java +++ b/src/main/java/com/salesforce/phoenix/coprocessor/GroupedAggregateRegionObserver.java @@ -27,10 +27,21 @@ ******************************************************************************/ package com.salesforce.phoenix.coprocessor; -import static com.salesforce.phoenix.query.QueryConstants.*; +import static com.salesforce.phoenix.query.QueryConstants.AGG_TIMESTAMP; +import static com.salesforce.phoenix.query.QueryConstants.SINGLE_COLUMN; +import static com.salesforce.phoenix.query.QueryConstants.SINGLE_COLUMN_FAMILY; -import java.io.*; -import java.util.*; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; @@ -38,7 +49,9 @@ import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.regionserver.*; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl; +import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.WritableUtils; import org.slf4j.Logger; @@ -54,7 +67,10 @@ import com.salesforce.phoenix.memory.MemoryManager.MemoryChunk; import com.salesforce.phoenix.query.QueryConstants; import com.salesforce.phoenix.schema.tuple.MultiKeyValueTuple; -import com.salesforce.phoenix.util.*; +import com.salesforce.phoenix.util.KeyValueUtil; +import com.salesforce.phoenix.util.ScanUtil; +import com.salesforce.phoenix.util.SizedUtil; +import com.salesforce.phoenix.util.TupleUtil; @@ -90,8 +106,7 @@ public class GroupedAggregateRegionObserver extends BaseScannerRegionObserver { * a 3 * b 1 * - * The client is required to do a final aggregation, since multiple rows with the same key may be returned from different regions. The returned rows - * are in sorted order. + * The client is required to do a sort and a final aggregation, since multiple rows with the same key may be returned from different regions. */ @Override protected RegionScanner doPostScannerOpen(ObserverContext c, Scan scan, RegionScanner s) throws IOException { @@ -119,7 +134,7 @@ protected RegionScanner doPostScannerOpen(ObserverContext deserializeGroupByExpressions(byte[] expressionBytes) t /** * Used for an aggregate query in which the key order does not necessarily match the group by key order. In this case, - * we must collect all distinct groups within a region into a map, aggregating as we go, and then at the end of the - * underlying scan, sort them and return them one by one during iteration. + * we must collect all distinct groups within a region into a map, aggregating as we go. */ private RegionScanner scanUnordered(ObserverContext c, Scan scan, final RegionScanner s, List expressions, ServerAggregators aggregators) throws IOException { @@ -259,7 +273,10 @@ private RegionScanner scanUnordered(ObserverContext getSplits() { return splits; } + private static class OrderingResultIteratorFactory implements ParallelIteratorFactory { + private final QueryServices services; + + public OrderingResultIteratorFactory(QueryServices services) { + this.services = services; + } + @Override + public PeekingResultIterator newIterator(ResultIterator scanner) throws SQLException { + Expression expression = RowKeyExpression.INSTANCE; + OrderByExpression orderByExpression = new OrderByExpression(expression, false, true); + int threshold = services.getProps().getInt(QueryServices.SPOOL_THRESHOLD_BYTES_ATTRIB, QueryServicesOptions.DEFAULT_SPOOL_THRESHOLD_BYTES); + return new OrderedResultIterator(scanner, Collections.singletonList(orderByExpression), threshold); + } + } + + private static class WrappingResultIteratorFactory implements ParallelIteratorFactory { + private final ParallelIteratorFactory innerFactory; + private final ParallelIteratorFactory outerFactory; + + public WrappingResultIteratorFactory(ParallelIteratorFactory innerFactory, ParallelIteratorFactory outerFactory) { + this.innerFactory = innerFactory; + this.outerFactory = outerFactory; + } + @Override + public PeekingResultIterator newIterator(ResultIterator scanner) throws SQLException { + PeekingResultIterator iterator = innerFactory.newIterator(scanner); + return outerFactory.newIterator(iterator); + } + } + + private ParallelIteratorFactory wrapParallelIteratorFactory () { + ParallelIteratorFactory innerFactory; + QueryServices services = context.getConnection().getQueryServices(); + if (groupBy.isEmpty() || groupBy.isOrderPreserving()) { + innerFactory = new SpoolingResultIterator.SpoolingResultIteratorFactory(services); + } else { + innerFactory = new OrderingResultIteratorFactory(services); + } + if (parallelIteratorFactory == null) { + return innerFactory; + } + // wrap any existing parallelIteratorFactory + return new WrappingResultIteratorFactory(innerFactory, parallelIteratorFactory); + } + @Override protected Scanner newScanner(ConnectionQueryServices services) throws SQLException { // Hack to set state on scan to make upgrade happen @@ -84,7 +151,7 @@ protected Scanner newScanner(ConnectionQueryServices services) throws SQLExcepti if (groupBy.isEmpty()) { UngroupedAggregateRegionObserver.serializeIntoScan(context.getScan()); } - ParallelIterators parallelIterators = new ParallelIterators(context, tableRef, statement, projection, groupBy, null, parallelIteratorFactory); + ParallelIterators parallelIterators = new ParallelIterators(context, tableRef, statement, projection, groupBy, null, wrapParallelIteratorFactory()); splits = parallelIterators.getSplits(); AggregatingResultIterator aggResultIterator; diff --git a/src/main/java/com/salesforce/phoenix/execute/ScanPlan.java b/src/main/java/com/salesforce/phoenix/execute/ScanPlan.java index d36007e2..dbee5612 100644 --- a/src/main/java/com/salesforce/phoenix/execute/ScanPlan.java +++ b/src/main/java/com/salesforce/phoenix/execute/ScanPlan.java @@ -33,13 +33,28 @@ import com.salesforce.phoenix.compile.GroupByCompiler.GroupBy; import com.salesforce.phoenix.compile.OrderByCompiler.OrderBy; -import com.salesforce.phoenix.compile.*; +import com.salesforce.phoenix.compile.RowProjector; +import com.salesforce.phoenix.compile.StatementContext; import com.salesforce.phoenix.coprocessor.ScanRegionObserver; -import com.salesforce.phoenix.iterate.*; +import com.salesforce.phoenix.iterate.ConcatResultIterator; +import com.salesforce.phoenix.iterate.LimitingResultIterator; +import com.salesforce.phoenix.iterate.MergeSortRowKeyResultIterator; +import com.salesforce.phoenix.iterate.MergeSortTopNResultIterator; +import com.salesforce.phoenix.iterate.ParallelIterators; import com.salesforce.phoenix.iterate.ParallelIterators.ParallelIteratorFactory; +import com.salesforce.phoenix.iterate.ResultIterator; +import com.salesforce.phoenix.iterate.SpoolingResultIterator; import com.salesforce.phoenix.parse.FilterableStatement; -import com.salesforce.phoenix.query.*; -import com.salesforce.phoenix.schema.*; +import com.salesforce.phoenix.query.ConnectionQueryServices; +import com.salesforce.phoenix.query.KeyRange; +import com.salesforce.phoenix.query.QueryConstants; +import com.salesforce.phoenix.query.QueryServices; +import com.salesforce.phoenix.query.QueryServicesOptions; +import com.salesforce.phoenix.query.Scanner; +import com.salesforce.phoenix.query.WrappedScanner; +import com.salesforce.phoenix.schema.PTable; +import com.salesforce.phoenix.schema.SaltingUtil; +import com.salesforce.phoenix.schema.TableRef; @@ -54,7 +69,7 @@ public class ScanPlan extends BasicQueryPlan { private List splits; public ScanPlan(StatementContext context, FilterableStatement statement, TableRef table, RowProjector projector, Integer limit, OrderBy orderBy, ParallelIteratorFactory parallelIteratorFactory) { - super(context, statement, table, projector, context.getBindManager().getParameterMetaData(), limit, orderBy, null, parallelIteratorFactory); + super(context, statement, table, projector, context.getBindManager().getParameterMetaData(), limit, orderBy, null, parallelIteratorFactory == null ? new SpoolingResultIterator.SpoolingResultIteratorFactory(context.getConnection().getQueryServices()) : parallelIteratorFactory); if (!orderBy.getOrderByExpressions().isEmpty()) { // TopN int thresholdBytes = context.getConnection().getQueryServices().getProps().getInt( QueryServices.SPOOL_THRESHOLD_BYTES_ATTRIB, QueryServicesOptions.DEFAULT_SPOOL_THRESHOLD_BYTES); diff --git a/src/main/java/com/salesforce/phoenix/expression/RowKeyExpression.java b/src/main/java/com/salesforce/phoenix/expression/RowKeyExpression.java new file mode 100644 index 00000000..518bb48a --- /dev/null +++ b/src/main/java/com/salesforce/phoenix/expression/RowKeyExpression.java @@ -0,0 +1,25 @@ +package com.salesforce.phoenix.expression; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; + +import com.salesforce.phoenix.schema.PDataType; +import com.salesforce.phoenix.schema.tuple.Tuple; + +public class RowKeyExpression extends BaseTerminalExpression { + public static final RowKeyExpression INSTANCE = new RowKeyExpression(); + + private RowKeyExpression() { + } + + @Override + public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { + tuple.getKey(ptr); + return true; + } + + @Override + public PDataType getDataType() { + return PDataType.VARBINARY; + } + +} diff --git a/src/main/java/com/salesforce/phoenix/iterate/OrderedResultIterator.java b/src/main/java/com/salesforce/phoenix/iterate/OrderedResultIterator.java index 429bf0b3..7aa56416 100644 --- a/src/main/java/com/salesforce/phoenix/iterate/OrderedResultIterator.java +++ b/src/main/java/com/salesforce/phoenix/iterate/OrderedResultIterator.java @@ -55,7 +55,7 @@ * @author syyang, jtaylor * @since 0.1 */ -public class OrderedResultIterator implements ResultIterator { +public class OrderedResultIterator implements PeekingResultIterator { /** A container that holds pointers to a {@link Result} and its sort keys. */ protected static class ResultEntry { @@ -104,7 +104,7 @@ public Expression apply(OrderByExpression column) { private final List orderByExpressions; private final long estimatedByteSize; - private ResultIterator resultIterator; + private PeekingResultIterator resultIterator; private long byteSize; protected ResultIterator getDelegate() { @@ -179,7 +179,7 @@ public Tuple next() throws SQLException { return getResultIterator().next(); } - private ResultIterator getResultIterator() throws SQLException { + private PeekingResultIterator getResultIterator() throws SQLException { if (resultIterator != null) { return resultIterator; } @@ -189,18 +189,34 @@ private ResultIterator getResultIterator() throws SQLException { final Comparator comparator = buildComparator(orderByExpressions); try{ final MappedByteBufferSortedQueue queueEntries = new MappedByteBufferSortedQueue(comparator, limit, thresholdBytes); - resultIterator = new BaseResultIterator() { + resultIterator = new PeekingResultIterator() { int count = 0; @Override public Tuple next() throws SQLException { ResultEntry entry = queueEntries.poll(); if (entry == null || (limit != null && ++count > limit)) { - resultIterator = ResultIterator.EMPTY_ITERATOR; + resultIterator = PeekingResultIterator.EMPTY_ITERATOR; + return null; + } + return entry.getResult(); + } + + @Override + public Tuple peek() throws SQLException { + if (limit != null && count > limit) { + return null; + } + ResultEntry entry = queueEntries.peek(); + if (entry == null) { return null; } return entry.getResult(); } + @Override + public void explain(List planSteps) { + } + @Override public void close() throws SQLException { queueEntries.close(); @@ -227,9 +243,14 @@ public void close() throws SQLException { return resultIterator; } + @Override + public Tuple peek() throws SQLException { + return getResultIterator().peek(); + } + @Override public void close() { - resultIterator = ResultIterator.EMPTY_ITERATOR; + resultIterator = PeekingResultIterator.EMPTY_ITERATOR; } diff --git a/src/test/java/com/salesforce/hbase/index/TestIndexWriter.java b/src/test/java/com/salesforce/hbase/index/TestIndexWriter.java index faad8f36..8993e3e1 100644 --- a/src/test/java/com/salesforce/hbase/index/TestIndexWriter.java +++ b/src/test/java/com/salesforce/hbase/index/TestIndexWriter.java @@ -114,6 +114,7 @@ public void correctlyCleanupResources() throws Exception { * all index writes for a mutation/batch are completed. * @throws Exception on failure */ + @SuppressWarnings("unchecked") @Test public void testSynchronouslyCompletesAllWrites() throws Exception { LOG.info("Starting " + testName.getTableNameString()); @@ -164,7 +165,8 @@ public Void answer(InvocationOnMock invocation) throws Throwable { * the batch write, so we should never see a failure here. * @throws Exception on failure */ - // @Ignore + @SuppressWarnings("unchecked") +// @Ignore @Test public void testFailureOnRunningUpdateAbortsPending() throws Exception { Abortable abort = new StubAbortable(); @@ -235,6 +237,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable { * that we correctly end the task * @throws Exception on failure */ + @SuppressWarnings("unchecked") @Test public void testShutdownInterruptsAsExpected() throws Exception { diff --git a/src/test/java/com/salesforce/phoenix/end2end/QueryExecTest.java b/src/test/java/com/salesforce/phoenix/end2end/QueryExecTest.java index a87f9cbd..e3273f0b 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/QueryExecTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/QueryExecTest.java @@ -27,6 +27,7 @@ ******************************************************************************/ package com.salesforce.phoenix.end2end; +import static com.salesforce.phoenix.util.TestUtil.ATABLE_NAME; import static com.salesforce.phoenix.util.TestUtil.A_VALUE; import static com.salesforce.phoenix.util.TestUtil.B_VALUE; import static com.salesforce.phoenix.util.TestUtil.C_VALUE; @@ -64,6 +65,8 @@ import java.util.Properties; import java.util.Set; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; @@ -71,6 +74,7 @@ import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; +import com.salesforce.phoenix.jdbc.PhoenixConnection; import com.salesforce.phoenix.jdbc.PhoenixStatement; import com.salesforce.phoenix.query.KeyRange; import com.salesforce.phoenix.schema.ConstraintViolationException; @@ -2927,4 +2931,67 @@ public void testCastOperatorInWhere() throws Exception { conn.close(); } } + + @Test + public void testSplitWithCachedMeta() throws Exception { + long ts = nextTimestamp(); + String tenantId = getOrganizationId(); + initATableValues(tenantId, getDefaultSplits(tenantId), null, ts); + // Tests that you don't get an ambiguous column exception when using the same alias as the column name + String query = "SELECT a_string, b_string, count(1) FROM atable WHERE organization_id=? and entity_id<=? GROUP BY a_string,b_string"; + Properties props = new Properties(TEST_PROPERTIES); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 + Connection conn = DriverManager.getConnection(PHOENIX_JDBC_URL, props); + HBaseAdmin admin = null; + try { + PreparedStatement statement = conn.prepareStatement(query); + statement.setString(1, tenantId); + statement.setString(2,ROW4); + ResultSet rs = statement.executeQuery(); + assertTrue(rs.next()); + assertEquals(A_VALUE, rs.getString(1)); + assertEquals(B_VALUE, rs.getString(2)); + assertEquals(2, rs.getLong(3)); + assertTrue(rs.next()); + assertEquals(A_VALUE, rs.getString(1)); + assertEquals(C_VALUE, rs.getString(2)); + assertEquals(1, rs.getLong(3)); + assertTrue(rs.next()); + assertEquals(A_VALUE, rs.getString(1)); + assertEquals(E_VALUE, rs.getString(2)); + assertEquals(1, rs.getLong(3)); + assertFalse(rs.next()); + + byte[] tableName = Bytes.toBytes(ATABLE_NAME); + admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin(); + HTable htable = (HTable)conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(tableName); + htable.clearRegionCache(); + int nRegions = htable.getRegionLocations().size(); + admin.split(tableName, ByteUtil.concat(Bytes.toBytes(tenantId), Bytes.toBytes("00A3"))); + do { + Thread.sleep(2000); + htable.clearRegionCache(); + } while (htable.getRegionLocations().size() == nRegions); + + statement.setString(1, tenantId); + rs = statement.executeQuery(); + assertTrue(rs.next()); + assertEquals(A_VALUE, rs.getString(1)); + assertEquals(B_VALUE, rs.getString(2)); + assertEquals(2, rs.getLong(3)); + assertTrue(rs.next()); + assertEquals(A_VALUE, rs.getString(1)); + assertEquals(C_VALUE, rs.getString(2)); + assertEquals(1, rs.getLong(3)); + assertTrue(rs.next()); + assertEquals(A_VALUE, rs.getString(1)); + assertEquals(E_VALUE, rs.getString(2)); + assertEquals(1, rs.getLong(3)); + assertFalse(rs.next()); + + } finally { + conn.close(); + } + } + } From 5fa52f04d5f5305fdc691eaca34fbb7ac364d4d4 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sun, 22 Sep 2013 17:31:38 -0700 Subject: [PATCH 062/102] Perf experiment - tweak priming of cache to match what Jesse does --- .../java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java index 0657fd68..b380c60d 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java @@ -53,6 +53,7 @@ public void batchStarted(MiniBatchOperationInProgress> m } ScanRanges scanRanges = ScanRanges.create(Collections.singletonList(keys), SaltingUtil.VAR_BINARY_SCHEMA); Scan scan = new Scan(); + scan.setRaw(true); // Project into scan only the columns we need to build the new index row and // delete the old index row. We use the columns that we pass through for // the Delete use case, as it includes indexed and covered columns as well @@ -62,7 +63,7 @@ public void batchStarted(MiniBatchOperationInProgress> m IndexMaintainer maintainer = maintainers.get(i); for (int j = 0; j < maintainer.getAllColumns().size(); j++) { ColumnReference ref = maintainer.getAllColumns().get(j); - scan.addColumn(ref.getFamily(), ref.getQualifier()); + scan.addFamily(ref.getFamily()); } } scan.setFilter(scanRanges.getSkipScanFilter()); From 67326825a396c4c85f5ff87ac41fb4a7ccac7f73 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Mon, 23 Sep 2013 01:39:43 -0700 Subject: [PATCH 063/102] Use HConnection region cache instead of our own --- .../phoenix/cache/ServerCacheClient.java | 42 ++- ...DefaultParallelIteratorRegionSplitter.java | 78 +++-- .../phoenix/iterate/ParallelIterators.java | 35 +- ...ipRangeParallelIteratorRegionSplitter.java | 27 +- .../query/ConnectionQueryServices.java | 14 +- .../query/ConnectionQueryServicesImpl.java | 77 ++--- .../ConnectionlessQueryServicesImpl.java | 7 +- .../DelegateConnectionQueryServices.java | 22 +- .../phoenix/query/QueryServices.java | 92 +----- .../phoenix/query/QueryServicesOptions.java | 13 +- ...ltParallelIteratorsRegionSplitterTest.java | 12 +- ...ngeParallelIteratorRegionSplitterTest.java | 12 +- .../end2end/index/ImmutableIndexTest.java | 300 ++++++++---------- .../end2end/index/MutableSaltedIndexTest.java | 14 - 14 files changed, 307 insertions(+), 438 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/cache/ServerCacheClient.java b/src/main/java/com/salesforce/phoenix/cache/ServerCacheClient.java index 173aac1c..3e33fd1c 100644 --- a/src/main/java/com/salesforce/phoenix/cache/ServerCacheClient.java +++ b/src/main/java/com/salesforce/phoenix/cache/ServerCacheClient.java @@ -34,8 +34,6 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Map; -import java.util.NavigableMap; import java.util.Random; import java.util.Set; import java.util.concurrent.ExecutorService; @@ -44,8 +42,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Scan; @@ -114,9 +111,9 @@ public TableRef getTableRef() { public class ServerCache implements SQLCloseable { private final int size; private final byte[] id; - private final ImmutableSet servers; + private final ImmutableSet servers; - public ServerCache(byte[] id, Set servers, int size) { + public ServerCache(byte[] id, Set servers, int size) { this.id = id; this.servers = ImmutableSet.copyOf(servers); this.size = size; @@ -163,17 +160,17 @@ public ServerCache addServerCache(ScanRanges keyRanges, final ImmutableBytesWrit ExecutorService executor = services.getExecutor(); List> futures = Collections.emptyList(); try { - NavigableMap locations = services.getAllTableRegions(cacheUsingTableRef); + List locations = services.getAllTableRegions(cacheUsingTableRef.getTable().getName().getBytes()); int nRegions = locations.size(); // Size these based on worst case futures = new ArrayList>(nRegions); - Set servers = new HashSet(nRegions); - for (Map.Entry entry : locations.entrySet()) { + Set servers = new HashSet(nRegions); + for (HRegionLocation entry : locations) { // Keep track of servers we've sent to and only send once - if ( ! servers.contains(entry.getValue()) && - keyRanges.intersect(entry.getKey().getStartKey(), entry.getKey().getEndKey())) { // Call RPC once per server - servers.add(entry.getValue()); - final byte[] key = entry.getKey().getStartKey(); + if ( ! servers.contains(entry) && + keyRanges.intersect(entry.getRegionInfo().getStartKey(), entry.getRegionInfo().getEndKey())) { // Call RPC once per server + servers.add(entry); + final byte[] key = entry.getRegionInfo().getStartKey(); final HTableInterface htable = services.getTable(cacheUsingTableRef.getTable().getName().getBytes()); closeables.add(htable); futures.add(executor.submit(new JobCallable() { @@ -241,28 +238,29 @@ public Object getJobId() { * @throws SQLException * @throws IllegalStateException if hashed table cannot be removed on any region server on which it was added */ - private void removeServerCache(byte[] cacheId, Set servers) throws SQLException { + private void removeServerCache(byte[] cacheId, Set servers) throws SQLException { ConnectionQueryServices services = connection.getQueryServices(); Throwable lastThrowable = null; - HTableInterface iterateOverTable = services.getTable(cacheUsingTableRef.getTable().getName().getBytes()); - NavigableMap locations = services.getAllTableRegions(cacheUsingTableRef); - Set remainingOnServers = new HashSet(servers); + byte[] tableName = cacheUsingTableRef.getTable().getName().getBytes(); + HTableInterface iterateOverTable = services.getTable(tableName); + List locations = services.getAllTableRegions(tableName); + Set remainingOnServers = new HashSet(servers); /** * Allow for the possibility that the region we based where to send our cache has split and been * relocated to another region server *after* we sent it, but before we removed it. To accommodate * this, we iterate through the current metadata boundaries and remove the cache once for each * server that we originally sent to. */ - for (Map.Entry entry : locations.entrySet()) { - if (remainingOnServers.contains(entry.getValue())) { // Call once per server + for (HRegionLocation entry : locations) { + if (remainingOnServers.contains(entry)) { // Call once per server try { - byte[] key = entry.getKey().getStartKey(); + byte[] key = entry.getRegionInfo().getStartKey(); ServerCachingProtocol protocol = iterateOverTable.coprocessorProxy(ServerCachingProtocol.class, key); protocol.removeServerCache(connection.getTenantId(), cacheId); - remainingOnServers.remove(entry.getValue()); + remainingOnServers.remove(entry); } catch (Throwable t) { lastThrowable = t; - LOG.error("Error trying to remove hash cache for " + entry.getValue(), t); + LOG.error("Error trying to remove hash cache for " + entry, t); } } } diff --git a/src/main/java/com/salesforce/phoenix/iterate/DefaultParallelIteratorRegionSplitter.java b/src/main/java/com/salesforce/phoenix/iterate/DefaultParallelIteratorRegionSplitter.java index 1a7367fc..ffe9de1b 100644 --- a/src/main/java/com/salesforce/phoenix/iterate/DefaultParallelIteratorRegionSplitter.java +++ b/src/main/java/com/salesforce/phoenix/iterate/DefaultParallelIteratorRegionSplitter.java @@ -28,20 +28,28 @@ package com.salesforce.phoenix.iterate; import java.sql.SQLException; -import java.util.*; -import java.util.Map.Entry; +import java.util.Collection; +import java.util.Collections; +import java.util.List; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.Bytes; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; -import com.google.common.collect.*; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Iterables; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Lists; import com.salesforce.phoenix.compile.StatementContext; import com.salesforce.phoenix.parse.HintNode; import com.salesforce.phoenix.parse.HintNode.Hint; -import com.salesforce.phoenix.query.*; +import com.salesforce.phoenix.query.KeyRange; +import com.salesforce.phoenix.query.QueryServices; +import com.salesforce.phoenix.query.QueryServicesOptions; +import com.salesforce.phoenix.query.StatsManager; import com.salesforce.phoenix.schema.TableRef; import com.salesforce.phoenix.util.ReadOnlyProps; @@ -59,7 +67,7 @@ public class DefaultParallelIteratorRegionSplitter implements ParallelIteratorRe protected final int maxConcurrency; protected final int maxIntraRegionParallelization; protected final StatementContext context; - protected final TableRef table; + protected final TableRef tableRef; public static DefaultParallelIteratorRegionSplitter getInstance(StatementContext context, TableRef table, HintNode hintNode) { return new DefaultParallelIteratorRegionSplitter(context, table, hintNode); @@ -67,7 +75,7 @@ public static DefaultParallelIteratorRegionSplitter getInstance(StatementContext protected DefaultParallelIteratorRegionSplitter(StatementContext context, TableRef table, HintNode hintNode) { this.context = context; - this.table = table; + this.tableRef = table; ReadOnlyProps props = context.getConnection().getQueryServices().getProps(); this.targetConcurrency = props.getInt(QueryServices.TARGET_QUERY_CONCURRENCY_ATTRIB, QueryServicesOptions.DEFAULT_TARGET_QUERY_CONCURRENCY); @@ -81,9 +89,9 @@ protected DefaultParallelIteratorRegionSplitter(StatementContext context, TableR } // Get the mapping between key range and the regions that contains them. - protected List> getAllRegions() throws SQLException { + protected List getAllRegions() throws SQLException { Scan scan = context.getScan(); - NavigableMap allTableRegions = context.getConnection().getQueryServices().getAllTableRegions(table); + List allTableRegions = context.getConnection().getQueryServices().getAllTableRegions(tableRef.getTable().getName().getBytes()); return filterRegions(allTableRegions, scan.getStartRow(), scan.getStopRow()); } @@ -95,24 +103,24 @@ protected List> getAllRegions() throws SQLExcepti * @return regions that intersect with the key range given by the startKey and stopKey */ // exposed for tests - public static List> filterRegions(NavigableMap allTableRegions, byte[] startKey, byte[] stopKey) { - Iterable> regions; + public static List filterRegions(List allTableRegions, byte[] startKey, byte[] stopKey) { + Iterable regions; final KeyRange keyRange = KeyRange.getKeyRange(startKey, true, stopKey, false); if (keyRange == KeyRange.EVERYTHING_RANGE) { - regions = allTableRegions.entrySet(); - } else { - regions = Iterables.filter(allTableRegions.entrySet(), new Predicate>() { - @Override - public boolean apply(Map.Entry region) { - KeyRange regionKeyRange = KeyRange.getKeyRange(region.getKey().getStartKey(), region.getKey().getEndKey()); - return keyRange.intersect(regionKeyRange) != KeyRange.EMPTY_RANGE; - } - }); + return allTableRegions; } + + regions = Iterables.filter(allTableRegions, new Predicate() { + @Override + public boolean apply(HRegionLocation location) { + KeyRange regionKeyRange = KeyRange.getKeyRange(location.getRegionInfo().getStartKey(), location.getRegionInfo().getEndKey()); + return keyRange.intersect(regionKeyRange) != KeyRange.EMPTY_RANGE; + } + }); return Lists.newArrayList(regions); } - protected List genKeyRanges(List> regions) { + protected List genKeyRanges(List regions) { if (regions.isEmpty()) { return Collections.emptyList(); } @@ -142,17 +150,17 @@ protected List genKeyRanges(List> r splitsPerRegion = Math.min(splitsPerRegion, maxIntraRegionParallelization); // Create a multi-map of ServerName to List which we'll use to round robin from to ensure // that we keep each region server busy for each query. - ListMultimap keyRangesPerRegion = ArrayListMultimap.create(regions.size(),regions.size() * splitsPerRegion);; + ListMultimap keyRangesPerRegion = ArrayListMultimap.create(regions.size(),regions.size() * splitsPerRegion);; if (splitsPerRegion == 1) { - for (Map.Entry region : regions) { - keyRangesPerRegion.put(region.getValue(), ParallelIterators.TO_KEY_RANGE.apply(region)); + for (HRegionLocation region : regions) { + keyRangesPerRegion.put(region, ParallelIterators.TO_KEY_RANGE.apply(region)); } } else { // Maintain bucket for each server and then returns KeyRanges in round-robin // order to ensure all servers are utilized. - for (Map.Entry region : regions) { - byte[] startKey = region.getKey().getStartKey(); - byte[] stopKey = region.getKey().getEndKey(); + for (HRegionLocation region : regions) { + byte[] startKey = region.getRegionInfo().getStartKey(); + byte[] stopKey = region.getRegionInfo().getEndKey(); boolean lowerUnbound = Bytes.compareTo(startKey, HConstants.EMPTY_START_ROW) == 0; boolean upperUnbound = Bytes.compareTo(stopKey, HConstants.EMPTY_END_ROW) == 0; /* @@ -162,16 +170,16 @@ protected List genKeyRanges(List> r * of date). */ if (lowerUnbound) { - startKey = statsManager.getMinKey(table); + startKey = statsManager.getMinKey(tableRef); if (startKey == null) { - keyRangesPerRegion.put(region.getValue(),ParallelIterators.TO_KEY_RANGE.apply(region)); + keyRangesPerRegion.put(region,ParallelIterators.TO_KEY_RANGE.apply(region)); continue; } } if (upperUnbound) { - stopKey = statsManager.getMaxKey(table); + stopKey = statsManager.getMaxKey(tableRef); if (stopKey == null) { - keyRangesPerRegion.put(region.getValue(),ParallelIterators.TO_KEY_RANGE.apply(region)); + keyRangesPerRegion.put(region,ParallelIterators.TO_KEY_RANGE.apply(region)); continue; } } @@ -181,14 +189,14 @@ protected List genKeyRanges(List> r if (Bytes.compareTo(startKey, stopKey) >= 0 || (boundaries = Bytes.split(startKey, stopKey, splitsPerRegion - 1)) == null) { // Bytes.split may return null if the key space // between start and end key is too small - keyRangesPerRegion.put(region.getValue(),ParallelIterators.TO_KEY_RANGE.apply(region)); + keyRangesPerRegion.put(region,ParallelIterators.TO_KEY_RANGE.apply(region)); } else { - keyRangesPerRegion.put(region.getValue(),KeyRange.getKeyRange(lowerUnbound ? KeyRange.UNBOUND : boundaries[0], boundaries[1])); + keyRangesPerRegion.put(region,KeyRange.getKeyRange(lowerUnbound ? KeyRange.UNBOUND : boundaries[0], boundaries[1])); if (boundaries.length > 1) { for (int i = 1; i < boundaries.length-2; i++) { - keyRangesPerRegion.put(region.getValue(),KeyRange.getKeyRange(boundaries[i], true, boundaries[i+1], false)); + keyRangesPerRegion.put(region,KeyRange.getKeyRange(boundaries[i], true, boundaries[i+1], false)); } - keyRangesPerRegion.put(region.getValue(),KeyRange.getKeyRange(boundaries[boundaries.length-2], true, upperUnbound ? KeyRange.UNBOUND : boundaries[boundaries.length-1], false)); + keyRangesPerRegion.put(region,KeyRange.getKeyRange(boundaries[boundaries.length-2], true, upperUnbound ? KeyRange.UNBOUND : boundaries[boundaries.length-1], false)); } } } diff --git a/src/main/java/com/salesforce/phoenix/iterate/ParallelIterators.java b/src/main/java/com/salesforce/phoenix/iterate/ParallelIterators.java index 34eed139..f2a424ef 100644 --- a/src/main/java/com/salesforce/phoenix/iterate/ParallelIterators.java +++ b/src/main/java/com/salesforce/phoenix/iterate/ParallelIterators.java @@ -28,11 +28,18 @@ package com.salesforce.phoenix.iterate; import java.sql.SQLException; -import java.util.*; -import java.util.concurrent.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; import org.apache.hadoop.hbase.filter.PageFilter; @@ -43,14 +50,22 @@ import com.google.common.base.Function; import com.salesforce.phoenix.compile.GroupByCompiler.GroupBy; -import com.salesforce.phoenix.compile.*; +import com.salesforce.phoenix.compile.RowProjector; +import com.salesforce.phoenix.compile.StatementContext; import com.salesforce.phoenix.job.JobManager.JobCallable; import com.salesforce.phoenix.parse.FilterableStatement; import com.salesforce.phoenix.parse.HintNode; -import com.salesforce.phoenix.query.*; +import com.salesforce.phoenix.query.ConnectionQueryServices; +import com.salesforce.phoenix.query.KeyRange; +import com.salesforce.phoenix.query.QueryConstants; +import com.salesforce.phoenix.query.QueryServices; import com.salesforce.phoenix.schema.PTable; import com.salesforce.phoenix.schema.TableRef; -import com.salesforce.phoenix.util.*; +import com.salesforce.phoenix.util.ReadOnlyProps; +import com.salesforce.phoenix.util.SQLCloseables; +import com.salesforce.phoenix.util.ScanUtil; +import com.salesforce.phoenix.util.SchemaUtil; +import com.salesforce.phoenix.util.ServerUtil; /** @@ -72,10 +87,10 @@ public static interface ParallelIteratorFactory { private static final int DEFAULT_THREAD_TIMEOUT_MS = 60000; // 1min - static final Function, KeyRange> TO_KEY_RANGE = new Function, KeyRange>() { + static final Function TO_KEY_RANGE = new Function() { @Override - public KeyRange apply(Map.Entry region) { - return KeyRange.getKeyRange(region.getKey().getStartKey(), region.getKey().getEndKey()); + public KeyRange apply(HRegionLocation region) { + return KeyRange.getKeyRange(region.getRegionInfo().getStartKey(), region.getRegionInfo().getEndKey()); } }; diff --git a/src/main/java/com/salesforce/phoenix/iterate/SkipRangeParallelIteratorRegionSplitter.java b/src/main/java/com/salesforce/phoenix/iterate/SkipRangeParallelIteratorRegionSplitter.java index a1c09d7b..d94baa7e 100644 --- a/src/main/java/com/salesforce/phoenix/iterate/SkipRangeParallelIteratorRegionSplitter.java +++ b/src/main/java/com/salesforce/phoenix/iterate/SkipRangeParallelIteratorRegionSplitter.java @@ -28,10 +28,9 @@ package com.salesforce.phoenix.iterate; import java.sql.SQLException; -import java.util.*; +import java.util.List; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.HRegionLocation; import com.google.common.base.Predicate; import com.google.common.collect.Iterables; @@ -56,23 +55,23 @@ protected SkipRangeParallelIteratorRegionSplitter(StatementContext context, Tabl } @Override - protected List> getAllRegions() throws SQLException { - NavigableMap allTableRegions = context.getConnection().getQueryServices().getAllTableRegions(table); + protected List getAllRegions() throws SQLException { + List allTableRegions = context.getConnection().getQueryServices().getAllTableRegions(tableRef.getTable().getName().getBytes()); return filterRegions(allTableRegions, context.getScanRanges()); } - public static List> filterRegions(NavigableMap allTableRegions, final ScanRanges ranges) { - Iterable> regions; + public static List filterRegions(List allTableRegions, final ScanRanges ranges) { + Iterable regions; if (ranges == ScanRanges.EVERYTHING) { - regions = allTableRegions.entrySet(); - } else if (ranges == ScanRanges.NOTHING) { - return Lists.>newArrayList(); + return allTableRegions; + } else if (ranges == ScanRanges.NOTHING) { // TODO: why not emptyList? + return Lists.newArrayList(); } else { - regions = Iterables.filter(allTableRegions.entrySet(), - new Predicate>() { + regions = Iterables.filter(allTableRegions, + new Predicate() { @Override - public boolean apply(Map.Entry region) { - return ranges.intersect(region.getKey().getStartKey(), region.getKey().getEndKey()); + public boolean apply(HRegionLocation region) { + return ranges.intersect(region.getRegionInfo().getStartKey(), region.getRegionInfo().getEndKey()); } }); } diff --git a/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServices.java b/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServices.java index f4cb3d92..19747311 100644 --- a/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServices.java +++ b/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServices.java @@ -28,10 +28,15 @@ package com.salesforce.phoenix.query; import java.sql.SQLException; -import java.util.*; +import java.util.List; +import java.util.Map; +import java.util.Properties; -import org.apache.hadoop.hbase.*; -import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Pair; @@ -40,7 +45,6 @@ import com.salesforce.phoenix.execute.MutationState; import com.salesforce.phoenix.jdbc.PhoenixConnection; import com.salesforce.phoenix.schema.PTableType; -import com.salesforce.phoenix.schema.TableRef; public interface ConnectionQueryServices extends QueryServices, MetaDataMutated { @@ -65,7 +69,7 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated public StatsManager getStatsManager(); - public NavigableMap getAllTableRegions(TableRef table) throws SQLException; + public List getAllTableRegions(byte[] tableName) throws SQLException; public PhoenixConnection connect(String url, Properties info) throws SQLException; diff --git a/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java b/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java index 0b32fec3..de86a649 100644 --- a/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java +++ b/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java @@ -41,29 +41,23 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.NavigableMap; import java.util.Properties; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.ZooKeeperConnectionException; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.client.HTableInterface; -import org.apache.hadoop.hbase.client.MetaScanner; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; @@ -79,11 +73,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.cache.RemovalListener; -import com.google.common.cache.RemovalNotification; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -121,7 +110,6 @@ import com.salesforce.phoenix.schema.SaltingUtil; import com.salesforce.phoenix.schema.TableAlreadyExistsException; import com.salesforce.phoenix.schema.TableNotFoundException; -import com.salesforce.phoenix.schema.TableRef; import com.salesforce.phoenix.util.ByteUtil; import com.salesforce.phoenix.util.IndexUtil; import com.salesforce.phoenix.util.JDBCUtil; @@ -148,13 +136,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement // Lowest HBase version on the cluster. private int lowestClusterHBaseVersion = Integer.MAX_VALUE; - /** - * keep a cache of HRegionInfo objects - * TODO: if/when we delete HBase meta data for tables when we drop them, we'll need to invalidate - * this cache properly. - */ - private final LoadingCache> tableRegionCache; - /** * Construct a ConnectionQueryServicesImpl that represents a connection to an HBase * cluster. @@ -187,24 +168,6 @@ public ConnectionQueryServicesImpl(QueryServices services, ConnectionInfo connec int statsUpdateFrequencyMs = this.getProps().getInt(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS); int maxStatsAgeMs = this.getProps().getInt(QueryServices.MAX_STATS_AGE_MS_ATTRIB, QueryServicesOptions.DEFAULT_MAX_STATS_AGE_MS); this.statsManager = new StatsManagerImpl(this, statsUpdateFrequencyMs, maxStatsAgeMs); - /** - * keep a cache of HRegionInfo objects - */ - tableRegionCache = CacheBuilder.newBuilder(). - expireAfterAccess(this.getProps().getLong(QueryServices.REGION_BOUNDARY_CACHE_TTL_MS_ATTRIB,QueryServicesOptions.DEFAULT_REGION_BOUNDARY_CACHE_TTL_MS), TimeUnit.MILLISECONDS) - .removalListener(new RemovalListener>(){ - @Override - public void onRemoval(RemovalNotification> notification) { - logger.info("REMOVE: {}", notification.getKey()); - } - }) - .build(new CacheLoader>(){ - @Override - public NavigableMap load(TableRef key) throws Exception { - logger.info("LOAD: {}", key); - return MetaScanner.allTableRegions(config, key.getTable().getName().getBytes(), false); - } - }); } @Override @@ -288,10 +251,24 @@ public ConnectionQueryServices getChildQueryServices(ImmutableBytesWritable tena } @Override - public NavigableMap getAllTableRegions(TableRef table) throws SQLException { + public List getAllTableRegions(byte[] tableName) throws SQLException { + /* + * Use HConnection.getRegionLocation as it uses the cache in HConnection, while getting + * all region locations from the HTable doesn't. + */ try { - return tableRegionCache.get(table); - } catch (ExecutionException e) { + // We could surface the package projected HConnectionImplementation.getNumberOfCachedRegionLocations + // to get the sizing info we need, but this would require a new class in the same package and a cast + // to this implementation class, so it's probably not worth it. + List locations = Lists.newArrayList(); + byte[] currentKey = HConstants.EMPTY_START_ROW; + do { + HRegionLocation regionLocation = connection.getRegionLocation(tableName, currentKey, false); + locations.add(regionLocation); + currentKey = regionLocation.getRegionInfo().getEndKey(); + } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW)); + return locations; + } catch (IOException e) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.GET_TABLE_REGIONS_FAIL) .setRootCause(e).build().buildException(); } @@ -938,15 +915,15 @@ private void checkClientServerCompatibility() throws SQLException { boolean isIncompatible = false; int minHBaseVersion = Integer.MAX_VALUE; try { - NavigableMap regionInfoMap = MetaScanner.allTableRegions(config, TYPE_TABLE_NAME_BYTES, false); - Set serverMap = Sets.newHashSetWithExpectedSize(regionInfoMap.size()); - TreeMap regionMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); - List regionKeys = Lists.newArrayListWithExpectedSize(regionInfoMap.size()); - for (Map.Entry entry : regionInfoMap.entrySet()) { - if (!serverMap.contains(entry.getValue())) { - regionKeys.add(entry.getKey().getStartKey()); - regionMap.put(entry.getKey().getRegionName(), entry.getValue()); - serverMap.add(entry.getValue()); + List locations = this.getAllTableRegions(TYPE_TABLE_NAME_BYTES); + Set serverMap = Sets.newHashSetWithExpectedSize(locations.size()); + TreeMap regionMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); + List regionKeys = Lists.newArrayListWithExpectedSize(locations.size()); + for (HRegionLocation entry : locations) { + if (!serverMap.contains(entry)) { + regionKeys.add(entry.getRegionInfo().getStartKey()); + regionMap.put(entry.getRegionInfo().getRegionName(), entry); + serverMap.add(entry); } } final TreeMap results = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); @@ -967,7 +944,7 @@ public void update(byte[] region, byte[] row, Long value) { // This is the "phoenix.jar" is in-place, but server is out-of-sync with client case. if (!isCompatible(result.getValue())) { isIncompatible = true; - ServerName name = regionMap.get(result.getKey()); + HRegionLocation name = regionMap.get(result.getKey()); buf.append(name); buf.append(';'); } diff --git a/src/main/java/com/salesforce/phoenix/query/ConnectionlessQueryServicesImpl.java b/src/main/java/com/salesforce/phoenix/query/ConnectionlessQueryServicesImpl.java index 4759ca29..49e6891c 100644 --- a/src/main/java/com/salesforce/phoenix/query/ConnectionlessQueryServicesImpl.java +++ b/src/main/java/com/salesforce/phoenix/query/ConnectionlessQueryServicesImpl.java @@ -32,13 +32,11 @@ import java.sql.SQLException; import java.util.List; import java.util.Map; -import java.util.NavigableMap; import java.util.Properties; -import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Mutation; @@ -60,7 +58,6 @@ import com.salesforce.phoenix.schema.PTable; import com.salesforce.phoenix.schema.PTableImpl; import com.salesforce.phoenix.schema.PTableType; -import com.salesforce.phoenix.schema.TableRef; import com.salesforce.phoenix.util.PhoenixRuntime; import com.salesforce.phoenix.util.SchemaUtil; @@ -97,7 +94,7 @@ public StatsManager getStatsManager() { } @Override - public NavigableMap getAllTableRegions(TableRef table) throws SQLException { + public List getAllTableRegions(byte[] tableName) throws SQLException { throw new UnsupportedOperationException(); } diff --git a/src/main/java/com/salesforce/phoenix/query/DelegateConnectionQueryServices.java b/src/main/java/com/salesforce/phoenix/query/DelegateConnectionQueryServices.java index 47960665..725d119d 100644 --- a/src/main/java/com/salesforce/phoenix/query/DelegateConnectionQueryServices.java +++ b/src/main/java/com/salesforce/phoenix/query/DelegateConnectionQueryServices.java @@ -28,10 +28,15 @@ package com.salesforce.phoenix.query; import java.sql.SQLException; -import java.util.*; - -import org.apache.hadoop.hbase.*; -import org.apache.hadoop.hbase.client.*; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Pair; @@ -39,7 +44,10 @@ import com.salesforce.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult; import com.salesforce.phoenix.execute.MutationState; import com.salesforce.phoenix.jdbc.PhoenixConnection; -import com.salesforce.phoenix.schema.*; +import com.salesforce.phoenix.schema.PColumn; +import com.salesforce.phoenix.schema.PMetaData; +import com.salesforce.phoenix.schema.PTable; +import com.salesforce.phoenix.schema.PTableType; public class DelegateConnectionQueryServices extends DelegateQueryServices implements ConnectionQueryServices { @@ -69,8 +77,8 @@ public StatsManager getStatsManager() { } @Override - public NavigableMap getAllTableRegions(TableRef table) throws SQLException { - return getDelegate().getAllTableRegions(table); + public List getAllTableRegions(byte[] tableName) throws SQLException { + return getDelegate().getAllTableRegions(tableName); } @Override diff --git a/src/main/java/com/salesforce/phoenix/query/QueryServices.java b/src/main/java/com/salesforce/phoenix/query/QueryServices.java index c3016bae..52da8fbb 100644 --- a/src/main/java/com/salesforce/phoenix/query/QueryServices.java +++ b/src/main/java/com/salesforce/phoenix/query/QueryServices.java @@ -42,96 +42,8 @@ /** * * Interface to group together services needed during querying. The - * following parameters may be set in - * {@link org.apache.hadoop.conf.Configuration}: - *
    - *
  • phoenix.query.timeoutMs: number of milliseconds - * after which a query will timeout on the client. Defaults to - * {@link com.salesforce.phoenix.query.QueryServicesOptions#DEFAULT_THREAD_TIMEOUT_MS}.
    • - *
  • phoenix.query.keepAliveMs: when the number of - * threads is greater than the core in the client side thread pool - * executor, this is the maximum time in milliseconds that excess idle - * threads will wait for a new tasks before terminating. Defaults to - * {@link com.salesforce.phoenix.query.QueryServicesOptions#DEFAULT_KEEP_ALIVE_MS}.
    • - *
  • phoenix.query.threadPoolSize: number of threads - * in client side thread pool executor. As the number of machines/cores - * in the cluster grows, this value should be increased. Defaults to - * {@link com.salesforce.phoenix.query.QueryServicesOptions#DEFAULT_THREAD_POOL_SIZE}.
    • - *
  • phoenix.query.queueSize: max queue depth of the - * bounded round robin backing the client side thread pool executor, - * beyond which attempts to queue additional work cause the client to - * block. If zero, a SynchronousQueue is used of the bounded round - * robin queue. Defaults to - * {@link com.salesforce.phoenix.query.QueryServicesOptions#DEFAULT_QUEUE_SIZE}.
    • - *
  • phoenix.query.spoolThresholdBytes: threshold - * size in bytes after which results from parallel executed aggregate - * query results are spooled to disk. Defaults to - * {@link com.salesforce.phoenix.query.QueryServicesOptions#DEFAULT_SPOOL_THRESHOLD_BYTES}.
    • - *
  • phoenix.query.maxGlobalMemoryPercentage: percentage of total - * memory ({@link java.lang.Runtime.getRuntime()#totalMemory}) that all threads - * may use. Only course grain memory usage is tracked, mainly accounting for memory - * usage in the intermediate map built during group by aggregation. When this limit - * is reached the clients block attempting to get more memory, essentially throttling - * memory usage. Defaults to - * {@link com.salesforce.phoenix.query.QueryServicesOptions#DEFAULT_MAX_MEMORY_PERC}.
    • - *
  • phoenix.query.maxGlobalMemoryWaitMs: maximum - * amount of time that a client will block while waiting for more memory - * to become available. After this amount of time, a - * {@link com.salesforce.phoenix.memory.InsufficientMemoryException} is - * thrown. Defaults to - * {@link com.salesforce.phoenix.query.QueryServicesOptions#DEFAULT_MAX_MEMORY_WAIT_MS}.
    • - *
  • phoenix.query.maxTenantMemoryPercentage: maximum - * percentage of phoenix.query.maxGlobalMemoryPercentage that any one tenant - * is allowed to consume. After this percentage, a - * {@link com.salesforce.phoenix.memory.InsufficientMemoryException} is - * thrown. Defaults to - * {@link com.salesforce.phoenix.query.QueryServicesOptions#DEFAULT_MAX_TENANT_MEMORY_PERC}.
    • - *
  • phoenix.query.targetConcurrency: target concurrent - * threads to use for a query. It serves as a soft limit on the number of - * scans into which a query may be split. A hard limit is imposed by - * phoenix.query.maxConcurrency (which should not be exceeded by this - * value). Defaults to - * {@link com.salesforce.phoenix.query.QueryServicesOptions#DEFAULT_TARGET_QUERY_CONCURRENCY}.
    • - *
  • phoenix.query.maxConcurrency: maximum concurrent - * threads to use for a query. It servers as a hard limit on the number - * of scans into which a query may be split. A soft limit is imposed by - * phoenix.query.targetConcurrency. Defaults to - * {@link com.salesforce.phoenix.query.QueryServicesOptions#DEFAULT_MAX_QUERY_CONCURRENCY}.
    • - *
  • phoenix.query.dateFormat: default pattern to use - * for convertion of a date to/from a string, whether through the - * TO_CHAR() or TO_DATE() functions, or through - * resultSet.getString(). Defaults to - * {@link com.salesforce.phoenix.query.QueryServicesOptions#DEFAULT_DATE_FORMAT}.
    • - *
  • phoenix.query.statsUpdateFrequency: the frequency - * in milliseconds at which each table stats will be updated. Defaults to - * {@link com.salesforce.phoenix.query.QueryServicesOptions#DEFAULT_STATS_UPDATE_FREQ_MS}.
    • - *
  • phoenix.query.maxStatsAge: the maximum age of - * stats in milliseconds after which they no longer will be used (i.e. - * if the stats could not be updated for this length of time, the stats - * are considered too old and thus no longer accurate enough to use). - * Defaults to {@link com.salesforce.phoenix.query.QueryServicesOptions#DEFAULT_MAX_STATS_AGE_MS}.
    • - *
  • phoenix.mutate.maxSize: the maximum number of rows - * that may be collected in {@link com.salesforce.phoenix.execute.MutationState} - * before a commit or rollback must be called. For better performance and to - * circumvent this limit, set {@link java.sql.Connection#setAutoCommit(boolean)} - * to TRUE, in which case, mutations (upserts and deletes) are performed - * on the server side without returning data back to the client. Defaults - * to {@link com.salesforce.phoenix.query.QueryServicesOptions#DEFAULT_MAX_MUTATION_SIZE}.
    • - *
  • phoenix.mutate.upsertBatchSize: deprecated - use strong>phoenix.mutate.batchSize - * instead.
    • - *
  • phoenix.mutate.batchSize: the number of rows - * that are batched together and automatically committed during the execution - * of an UPSERT SELECT or DELETE statement. This property may be overridden at - * connection time by specifying a {@link com.salesforce.phoenix.util.PhoenixRuntime#UPSERT_BATCH_SIZE_ATTRIB} - * property value. Note that the connection property value does not affect the - * batch size used by the coprocessor when these statements are executed - * completely on the server side. Defaults to - * {@link com.salesforce.phoenix.query.QueryServicesOptions#DEFAULT_MUTATE_BATCH_SIZE}.
    • - *
  • phoenix.query.regionBoundaryCacheTTL: the time-to-live - * in milliseconds of the region boundary cache used to guide the split - * points for query parallelization. Defaults to - * {@link com.salesforce.phoenix.query.QueryServicesOptions#DEFAULT_REGION_BOUNDARY_CACHE_TTL_MS}.
    • - *
+ * parameters that may be set in {@link org.apache.hadoop.conf.Configuration} + * are documented here: https://github.com/forcedotcom/phoenix/wiki/Tuning * * @author jtaylor * @since 0.1 diff --git a/src/main/java/com/salesforce/phoenix/query/QueryServicesOptions.java b/src/main/java/com/salesforce/phoenix/query/QueryServicesOptions.java index d3ada876..791e2569 100644 --- a/src/main/java/com/salesforce/phoenix/query/QueryServicesOptions.java +++ b/src/main/java/com/salesforce/phoenix/query/QueryServicesOptions.java @@ -33,12 +33,12 @@ import static com.salesforce.phoenix.query.QueryServices.IMMUTABLE_ROWS_ATTRIB; import static com.salesforce.phoenix.query.QueryServices.KEEP_ALIVE_MS_ATTRIB; import static com.salesforce.phoenix.query.QueryServices.MASTER_INFO_PORT_ATTRIB; -import static com.salesforce.phoenix.query.QueryServices.MAX_SERVER_CACHE_SIZE_ATTRIB; import static com.salesforce.phoenix.query.QueryServices.MAX_INTRA_REGION_PARALLELIZATION_ATTRIB; import static com.salesforce.phoenix.query.QueryServices.MAX_MEMORY_PERC_ATTRIB; import static com.salesforce.phoenix.query.QueryServices.MAX_MEMORY_WAIT_MS_ATTRIB; import static com.salesforce.phoenix.query.QueryServices.MAX_MUTATION_SIZE_ATTRIB; import static com.salesforce.phoenix.query.QueryServices.MAX_QUERY_CONCURRENCY_ATTRIB; +import static com.salesforce.phoenix.query.QueryServices.MAX_SERVER_CACHE_SIZE_ATTRIB; import static com.salesforce.phoenix.query.QueryServices.MAX_SERVER_CACHE_TIME_TO_LIVE_MS; import static com.salesforce.phoenix.query.QueryServices.MAX_SPOOL_TO_DISK_BYTES_ATTRIB; import static com.salesforce.phoenix.query.QueryServices.MAX_TENANT_MEMORY_PERC_ATTRIB; @@ -46,7 +46,6 @@ import static com.salesforce.phoenix.query.QueryServices.QUEUE_SIZE_ATTRIB; import static com.salesforce.phoenix.query.QueryServices.REGIONSERVER_INFO_PORT_ATTRIB; import static com.salesforce.phoenix.query.QueryServices.REGIONSERVER_LEASE_PERIOD_ATTRIB; -import static com.salesforce.phoenix.query.QueryServices.REGION_BOUNDARY_CACHE_TTL_MS_ATTRIB; import static com.salesforce.phoenix.query.QueryServices.ROW_KEY_ORDER_SALTED_TABLE_ATTRIB; import static com.salesforce.phoenix.query.QueryServices.RPC_TIMEOUT_ATTRIB; import static com.salesforce.phoenix.query.QueryServices.SCAN_CACHE_SIZE_ATTRIB; @@ -95,7 +94,6 @@ public class QueryServicesOptions { public final static int DEFAULT_MUTATE_BATCH_SIZE = 1000; // Batch size for UPSERT SELECT and DELETE // The only downside of it being out-of-sync is that the parallelization of the scan won't be as balanced as it could be. - public static final int DEFAULT_REGION_BOUNDARY_CACHE_TTL_MS = 15000; // How long to cache region boundary info for parallelization calculation public static final int DEFAULT_MAX_SERVER_CACHE_TIME_TO_LIVE_MS = 30000; // 30 sec (with no activity) public static final int DEFAULT_SCAN_CACHE_SIZE = 1000; public static final int DEFAULT_MAX_INTRA_REGION_PARALLELIZATION = DEFAULT_MAX_QUERY_CONCURRENCY; @@ -145,7 +143,6 @@ public static QueryServicesOptions withDefaults() { .setIfUnset(STATS_UPDATE_FREQ_MS_ATTRIB, DEFAULT_STATS_UPDATE_FREQ_MS) .setIfUnset(CALL_QUEUE_ROUND_ROBIN_ATTRIB, DEFAULT_CALL_QUEUE_ROUND_ROBIN) .setIfUnset(MAX_MUTATION_SIZE_ATTRIB, DEFAULT_MAX_MUTATION_SIZE) - .setIfUnset(REGION_BOUNDARY_CACHE_TTL_MS_ATTRIB, DEFAULT_REGION_BOUNDARY_CACHE_TTL_MS) .setIfUnset(MAX_INTRA_REGION_PARALLELIZATION_ATTRIB, DEFAULT_MAX_INTRA_REGION_PARALLELIZATION) .setIfUnset(ROW_KEY_ORDER_SALTED_TABLE_ATTRIB, DEFAULT_ROW_KEY_ORDER_SALTED_TABLE) .setIfUnset(USE_INDEXES_ATTRIB, DEFAULT_USE_INDEXES) @@ -261,10 +258,6 @@ public QueryServicesOptions setMaxIntraRegionParallelization(int maxIntraRegionP return set(MAX_INTRA_REGION_PARALLELIZATION_ATTRIB, maxIntraRegionParallelization); } - public QueryServicesOptions setRegionBoundaryCacheTTLMs(int regionBoundaryCacheTTL) { - return set(REGION_BOUNDARY_CACHE_TTL_MS_ATTRIB, regionBoundaryCacheTTL); - } - public QueryServicesOptions setRowKeyOrderSaltedTable(boolean rowKeyOrderSaltedTable) { return set(ROW_KEY_ORDER_SALTED_TABLE_ATTRIB, rowKeyOrderSaltedTable); } @@ -321,10 +314,6 @@ public int getMaxIntraRegionParallelization() { return config.getInt(MAX_INTRA_REGION_PARALLELIZATION_ATTRIB, DEFAULT_MAX_INTRA_REGION_PARALLELIZATION); } - public int getRegionBoundaryCacheTTLMs() { - return config.getInt(REGION_BOUNDARY_CACHE_TTL_MS_ATTRIB, DEFAULT_REGION_BOUNDARY_CACHE_TTL_MS); - } - public boolean isUseIndexes() { return config.getBoolean(USE_INDEXES_ATTRIB, DEFAULT_USE_INDEXES); } diff --git a/src/test/java/com/salesforce/phoenix/end2end/DefaultParallelIteratorsRegionSplitterTest.java b/src/test/java/com/salesforce/phoenix/end2end/DefaultParallelIteratorsRegionSplitterTest.java index 664bc340..badc5932 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/DefaultParallelIteratorsRegionSplitterTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/DefaultParallelIteratorsRegionSplitterTest.java @@ -44,12 +44,10 @@ import java.util.Comparator; import java.util.List; import java.util.Map; -import java.util.NavigableMap; import java.util.Properties; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.Bytes; import org.junit.BeforeClass; @@ -133,13 +131,13 @@ private static TableRef getTableRef(Connection conn, long ts) throws SQLExceptio private static List getSplits(Connection conn, long ts, final Scan scan) throws SQLException { - TableRef table = getTableRef(conn, ts); + TableRef tableRef = getTableRef(conn, ts); PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); - final NavigableMap regions = pconn.getQueryServices().getAllTableRegions(table); + final List regions = pconn.getQueryServices().getAllTableRegions(tableRef.getTable().getName().getBytes()); StatementContext context = new StatementContext(SelectStatement.SELECT_ONE, pconn, null, Collections.emptyList(), scan); - DefaultParallelIteratorRegionSplitter splitter = new DefaultParallelIteratorRegionSplitter(context, table, HintNode.EMPTY_HINT_NODE) { + DefaultParallelIteratorRegionSplitter splitter = new DefaultParallelIteratorRegionSplitter(context, tableRef, HintNode.EMPTY_HINT_NODE) { @Override - protected List> getAllRegions() throws SQLException { + protected List getAllRegions() throws SQLException { return DefaultParallelIteratorRegionSplitter.filterRegions(regions, scan.getStartRow(), scan.getStopRow()); } }; diff --git a/src/test/java/com/salesforce/phoenix/end2end/SkipRangeParallelIteratorRegionSplitterTest.java b/src/test/java/com/salesforce/phoenix/end2end/SkipRangeParallelIteratorRegionSplitterTest.java index 9ea5937d..346587fe 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/SkipRangeParallelIteratorRegionSplitterTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/SkipRangeParallelIteratorRegionSplitterTest.java @@ -40,11 +40,9 @@ import java.util.Comparator; import java.util.List; import java.util.Map; -import java.util.NavigableMap; import java.util.Properties; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.Bytes; import org.junit.BeforeClass; @@ -115,12 +113,12 @@ public void testGetSplitsWithSkipScanFilter() throws Exception { Properties props = new Properties(TEST_PROPERTIES); Connection conn = DriverManager.getConnection(url, props); PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); - TableRef table = new TableRef(null,pconn.getPMetaData().getTable(SchemaUtil.getTableName(SCHEMA_NAME, TABLE_NAME)),ts, false); - NavigableMap regions = pconn.getQueryServices().getAllTableRegions(table); + TableRef tableRef = new TableRef(null,pconn.getPMetaData().getTable(SchemaUtil.getTableName(SCHEMA_NAME, TABLE_NAME)),ts, false); + List regions = pconn.getQueryServices().getAllTableRegions(tableRef.getTable().getName().getBytes()); conn.close(); initTableValues(); - List ranges = getSplits(table, scan, regions, scanRanges); + List ranges = getSplits(tableRef, scan, regions, scanRanges); assertEquals("Unexpected number of splits: " + ranges.size(), expectedSplits.size(), ranges.size()); for (int i=0; i getSplits(TableRef table, final Scan scan, final NavigableMap regions, + private static List getSplits(TableRef table, final Scan scan, final List regions, final ScanRanges scanRanges) throws SQLException { PhoenixConnection connection = DriverManager.getConnection(getUrl(), TEST_PROPERTIES).unwrap(PhoenixConnection.class); StatementContext context = new StatementContext(SelectStatement.SELECT_ONE, connection, null, Collections.emptyList(), scan); diff --git a/src/test/java/com/salesforce/phoenix/end2end/index/ImmutableIndexTest.java b/src/test/java/com/salesforce/phoenix/end2end/index/ImmutableIndexTest.java index b76b18d0..b8952165 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/index/ImmutableIndexTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/index/ImmutableIndexTest.java @@ -13,23 +13,19 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.Map; import java.util.Properties; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.BeforeClass; +import org.junit.Before; import org.junit.Test; -import com.google.common.collect.Maps; import com.salesforce.phoenix.end2end.BaseHBaseManagedTimeTest; import com.salesforce.phoenix.jdbc.PhoenixConnection; import com.salesforce.phoenix.query.ConnectionQueryServices; import com.salesforce.phoenix.query.QueryConstants; -import com.salesforce.phoenix.query.QueryServices; import com.salesforce.phoenix.util.QueryUtil; -import com.salesforce.phoenix.util.ReadOnlyProps; import com.salesforce.phoenix.util.SchemaUtil; @@ -39,15 +35,6 @@ public class ImmutableIndexTest extends BaseHBaseManagedTimeTest{ private static final byte[] DATA_TABLE_FULL_NAME = Bytes.toBytes(SchemaUtil.getTableName(null, "T")); private static final byte[] INDEX_TABLE_FULL_NAME = Bytes.toBytes(SchemaUtil.getTableName(null, "I")); - @BeforeClass - public static void doSetup() throws Exception { - Map props = Maps.newHashMapWithExpectedSize(1); - // Don't cache meta information for this test because the splits change between tests - props.put(QueryServices.REGION_BOUNDARY_CACHE_TTL_MS_ATTRIB, Integer.toString(0)); - // Must update config before starting server - startServer(getUrl(), new ReadOnlyProps(props.entrySet().iterator())); - } - // Populate the test table with data. private static void populateTestTable() throws SQLException { Properties props = new Properties(TEST_PROPERTIES); @@ -113,7 +100,8 @@ private static void populateTestTable() throws SQLException { } } - private static void destroyTables() throws Exception { + @Before + public void destroyTables() throws Exception { // Physically delete HBase table so that splits occur as expected for each test Properties props = new Properties(TEST_PROPERTIES); ConnectionQueryServices services = DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class).getQueryServices(); @@ -150,124 +138,120 @@ public void testImmutableTableIndexMaintanenceUnsalted() throws Exception { } private void testImmutableTableIndexMaintanence(Integer tableSaltBuckets, Integer indexSaltBuckets) throws Exception { - try { - String query; - ResultSet rs; - - Properties props = new Properties(TEST_PROPERTIES); - Connection conn = DriverManager.getConnection(getUrl(), props); - conn.setAutoCommit(false); - conn.createStatement().execute("CREATE TABLE t (k VARCHAR NOT NULL PRIMARY KEY, v VARCHAR) IMMUTABLE_ROWS=true " + (tableSaltBuckets == null ? "" : ", SALT_BUCKETS=" + tableSaltBuckets)); - query = "SELECT * FROM t"; - rs = conn.createStatement().executeQuery(query); - assertFalse(rs.next()); - - conn.createStatement().execute("CREATE INDEX i ON t (v DESC)" + (indexSaltBuckets == null ? "" : " SALT_BUCKETS=" + indexSaltBuckets)); - query = "SELECT * FROM i"; - rs = conn.createStatement().executeQuery(query); - assertFalse(rs.next()); - - PreparedStatement stmt = conn.prepareStatement("UPSERT INTO t VALUES(?,?)"); - stmt.setString(1,"a"); - stmt.setString(2, "x"); - stmt.execute(); - stmt.setString(1,"b"); - stmt.setString(2, "y"); - stmt.execute(); - conn.commit(); - - query = "SELECT * FROM i"; - rs = conn.createStatement().executeQuery(query); - assertTrue(rs.next()); - assertEquals("y",rs.getString(1)); - assertEquals("b",rs.getString(2)); - assertTrue(rs.next()); - assertEquals("x",rs.getString(1)); - assertEquals("a",rs.getString(2)); - assertFalse(rs.next()); - - query = "SELECT k,v FROM t WHERE v = 'y'"; - rs = conn.createStatement().executeQuery(query); - assertTrue(rs.next()); - assertEquals("b",rs.getString(1)); - assertEquals("y",rs.getString(2)); - assertFalse(rs.next()); - - String expectedPlan; - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - expectedPlan = indexSaltBuckets == null ? - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER I 'y'" : - ("CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 KEYS OVER I 0...3,'y'\n" + - "CLIENT MERGE SORT"); - assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); - - // Will use index, so rows returned in DESC order. - // This is not a bug, though, because we can - // return in any order. - query = "SELECT k,v FROM t WHERE v >= 'x'"; - rs = conn.createStatement().executeQuery(query); - assertTrue(rs.next()); - assertEquals("b",rs.getString(1)); - assertEquals("y",rs.getString(2)); - assertTrue(rs.next()); - assertEquals("a",rs.getString(1)); - assertEquals("x",rs.getString(2)); - assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - expectedPlan = indexSaltBuckets == null ? - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER I (*-'x']" : - ("CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 RANGES OVER I 0...3,(*-'x']\n" + - "CLIENT MERGE SORT"); - assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); - - // Will still use index, since there's no LIMIT clause - query = "SELECT k,v FROM t WHERE v >= 'x' ORDER BY k"; - rs = conn.createStatement().executeQuery(query); - assertTrue(rs.next()); - assertEquals("a",rs.getString(1)); - assertEquals("x",rs.getString(2)); - assertTrue(rs.next()); - assertEquals("b",rs.getString(1)); - assertEquals("y",rs.getString(2)); - assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - // Turns into an ORDER BY, which could be bad if lots of data is - // being returned. Without stats we don't know. The alternative - // would be a full table scan. - expectedPlan = indexSaltBuckets == null ? - ("CLIENT PARALLEL 1-WAY RANGE SCAN OVER I (*-'x']\n" + - " SERVER TOP -1 ROWS SORTED BY [:K]\n" + - "CLIENT MERGE SORT") : - ("CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 RANGES OVER I 0...3,(*-'x']\n" + - " SERVER TOP -1 ROWS SORTED BY [:K]\n" + - "CLIENT MERGE SORT"); - assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); - - // Will use data table now, since there's a LIMIT clause and - // we're able to optimize out the ORDER BY, unless the data - // table is salted. - query = "SELECT k,v FROM t WHERE v >= 'x' ORDER BY k LIMIT 2"; - rs = conn.createStatement().executeQuery(query); - assertTrue(rs.next()); - assertEquals("a",rs.getString(1)); - assertEquals("x",rs.getString(2)); - assertTrue(rs.next()); - assertEquals("b",rs.getString(1)); - assertEquals("y",rs.getString(2)); - assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - expectedPlan = tableSaltBuckets == null ? - "CLIENT PARALLEL 1-WAY FULL SCAN OVER T\n" + - " SERVER FILTER BY V >= 'x'\n" + - " SERVER 2 ROW LIMIT\n" + - "CLIENT 2 ROW LIMIT" : - "CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 RANGES OVER I 0...3,(*-'x']\n" + - " SERVER TOP 2 ROWS SORTED BY [:K]\n" + - "CLIENT MERGE SORT"; - assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); - } finally { - destroyTables(); - } + String query; + ResultSet rs; + + Properties props = new Properties(TEST_PROPERTIES); + Connection conn = DriverManager.getConnection(getUrl(), props); + conn.setAutoCommit(false); + conn.createStatement().execute("CREATE TABLE t (k VARCHAR NOT NULL PRIMARY KEY, v VARCHAR) IMMUTABLE_ROWS=true " + (tableSaltBuckets == null ? "" : ", SALT_BUCKETS=" + tableSaltBuckets)); + query = "SELECT * FROM t"; + rs = conn.createStatement().executeQuery(query); + assertFalse(rs.next()); + + conn.createStatement().execute("CREATE INDEX i ON t (v DESC)" + (indexSaltBuckets == null ? "" : " SALT_BUCKETS=" + indexSaltBuckets)); + query = "SELECT * FROM i"; + rs = conn.createStatement().executeQuery(query); + assertFalse(rs.next()); + + PreparedStatement stmt = conn.prepareStatement("UPSERT INTO t VALUES(?,?)"); + stmt.setString(1,"a"); + stmt.setString(2, "x"); + stmt.execute(); + stmt.setString(1,"b"); + stmt.setString(2, "y"); + stmt.execute(); + conn.commit(); + + query = "SELECT * FROM i"; + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("y",rs.getString(1)); + assertEquals("b",rs.getString(2)); + assertTrue(rs.next()); + assertEquals("x",rs.getString(1)); + assertEquals("a",rs.getString(2)); + assertFalse(rs.next()); + + query = "SELECT k,v FROM t WHERE v = 'y'"; + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("b",rs.getString(1)); + assertEquals("y",rs.getString(2)); + assertFalse(rs.next()); + + String expectedPlan; + rs = conn.createStatement().executeQuery("EXPLAIN " + query); + expectedPlan = indexSaltBuckets == null ? + "CLIENT PARALLEL 1-WAY RANGE SCAN OVER I 'y'" : + ("CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 KEYS OVER I 0...3,'y'\n" + + "CLIENT MERGE SORT"); + assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); + + // Will use index, so rows returned in DESC order. + // This is not a bug, though, because we can + // return in any order. + query = "SELECT k,v FROM t WHERE v >= 'x'"; + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("b",rs.getString(1)); + assertEquals("y",rs.getString(2)); + assertTrue(rs.next()); + assertEquals("a",rs.getString(1)); + assertEquals("x",rs.getString(2)); + assertFalse(rs.next()); + rs = conn.createStatement().executeQuery("EXPLAIN " + query); + expectedPlan = indexSaltBuckets == null ? + "CLIENT PARALLEL 1-WAY RANGE SCAN OVER I (*-'x']" : + ("CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 RANGES OVER I 0...3,(*-'x']\n" + + "CLIENT MERGE SORT"); + assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); + + // Will still use index, since there's no LIMIT clause + query = "SELECT k,v FROM t WHERE v >= 'x' ORDER BY k"; + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("a",rs.getString(1)); + assertEquals("x",rs.getString(2)); + assertTrue(rs.next()); + assertEquals("b",rs.getString(1)); + assertEquals("y",rs.getString(2)); + assertFalse(rs.next()); + rs = conn.createStatement().executeQuery("EXPLAIN " + query); + // Turns into an ORDER BY, which could be bad if lots of data is + // being returned. Without stats we don't know. The alternative + // would be a full table scan. + expectedPlan = indexSaltBuckets == null ? + ("CLIENT PARALLEL 1-WAY RANGE SCAN OVER I (*-'x']\n" + + " SERVER TOP -1 ROWS SORTED BY [:K]\n" + + "CLIENT MERGE SORT") : + ("CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 RANGES OVER I 0...3,(*-'x']\n" + + " SERVER TOP -1 ROWS SORTED BY [:K]\n" + + "CLIENT MERGE SORT"); + assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); + + // Will use data table now, since there's a LIMIT clause and + // we're able to optimize out the ORDER BY, unless the data + // table is salted. + query = "SELECT k,v FROM t WHERE v >= 'x' ORDER BY k LIMIT 2"; + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("a",rs.getString(1)); + assertEquals("x",rs.getString(2)); + assertTrue(rs.next()); + assertEquals("b",rs.getString(1)); + assertEquals("y",rs.getString(2)); + assertFalse(rs.next()); + rs = conn.createStatement().executeQuery("EXPLAIN " + query); + expectedPlan = tableSaltBuckets == null ? + "CLIENT PARALLEL 1-WAY FULL SCAN OVER T\n" + + " SERVER FILTER BY V >= 'x'\n" + + " SERVER 2 ROW LIMIT\n" + + "CLIENT 2 ROW LIMIT" : + "CLIENT PARALLEL 4-WAY SKIP SCAN ON 4 RANGES OVER I 0...3,(*-'x']\n" + + " SERVER TOP 2 ROWS SORTED BY [:K]\n" + + "CLIENT MERGE SORT"; + assertEquals(expectedPlan,QueryUtil.getExplainPlan(rs)); } @Test @@ -275,32 +259,28 @@ public void testIndexWithNullableFixedWithCols() throws Exception { Properties props = new Properties(TEST_PROPERTIES); Connection conn = DriverManager.getConnection(getUrl(), props); conn.setAutoCommit(false); - try { - ensureTableCreated(getUrl(), INDEX_DATA_TABLE); - populateTestTable(); - String ddl = "CREATE INDEX IDX ON " + INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + INDEX_DATA_TABLE - + " (char_col1 ASC, int_col1 ASC)" - + " INCLUDE (long_col1, long_col2)"; - PreparedStatement stmt = conn.prepareStatement(ddl); - stmt.execute(); - - String query = "SELECT char_col1, int_col1 from " + INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + INDEX_DATA_TABLE; - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER INDEX_TEST.IDX", QueryUtil.getExplainPlan(rs)); - - rs = conn.createStatement().executeQuery(query); - assertTrue(rs.next()); - assertEquals("chara", rs.getString(1)); - assertEquals(2, rs.getInt(2)); - assertTrue(rs.next()); - assertEquals("chara", rs.getString(1)); - assertEquals(3, rs.getInt(2)); - assertTrue(rs.next()); - assertEquals("chara", rs.getString(1)); - assertEquals(4, rs.getInt(2)); - assertFalse(rs.next()); - } finally { - conn.close(); - } + ensureTableCreated(getUrl(), INDEX_DATA_TABLE); + populateTestTable(); + String ddl = "CREATE INDEX IDX ON " + INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + INDEX_DATA_TABLE + + " (char_col1 ASC, int_col1 ASC)" + + " INCLUDE (long_col1, long_col2)"; + PreparedStatement stmt = conn.prepareStatement(ddl); + stmt.execute(); + + String query = "SELECT char_col1, int_col1 from " + INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + INDEX_DATA_TABLE; + ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + query); + assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER INDEX_TEST.IDX", QueryUtil.getExplainPlan(rs)); + + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("chara", rs.getString(1)); + assertEquals(2, rs.getInt(2)); + assertTrue(rs.next()); + assertEquals("chara", rs.getString(1)); + assertEquals(3, rs.getInt(2)); + assertTrue(rs.next()); + assertEquals("chara", rs.getString(1)); + assertEquals(4, rs.getInt(2)); + assertFalse(rs.next()); } } diff --git a/src/test/java/com/salesforce/phoenix/end2end/index/MutableSaltedIndexTest.java b/src/test/java/com/salesforce/phoenix/end2end/index/MutableSaltedIndexTest.java index f36d49a6..f45111e5 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/index/MutableSaltedIndexTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/index/MutableSaltedIndexTest.java @@ -9,36 +9,22 @@ import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.util.Map; import java.util.Properties; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; -import com.google.common.collect.Maps; import com.salesforce.phoenix.jdbc.PhoenixConnection; import com.salesforce.phoenix.query.ConnectionQueryServices; -import com.salesforce.phoenix.query.QueryServices; import com.salesforce.phoenix.util.QueryUtil; -import com.salesforce.phoenix.util.ReadOnlyProps; public class MutableSaltedIndexTest extends BaseMutableIndexTest{ private static final int TABLE_SPLITS = 3; private static final int INDEX_SPLITS = 4; - @BeforeClass - public static void doSetup() throws Exception { - Map props = Maps.newHashMapWithExpectedSize(1); - // Don't cache meta information for this test because the splits change between tests - props.put(QueryServices.REGION_BOUNDARY_CACHE_TTL_MS_ATTRIB, Integer.toString(0)); - // Must update config before starting server - startServer(getUrl(), new ReadOnlyProps(props.entrySet().iterator())); - } - @Before public void destroyTables() throws Exception { // Physically delete HBase table so that splits occur as expected for each test From 2582e65141825c5755205057cfdc8c855c6e019e Mon Sep 17 00:00:00 2001 From: James Taylor Date: Mon, 23 Sep 2013 15:41:53 -0700 Subject: [PATCH 064/102] Issue proper Delete when only covered column changed --- .../hbase/index/util/IndexManagementUtil.java | 21 ++++- .../phoenix/index/IndexMaintainer.java | 89 ++++++++++--------- .../index/IndexMetaDataCacheClient.java | 2 +- .../phoenix/index/PhoenixIndexCodec.java | 4 +- .../salesforce/phoenix/util/IndexUtil.java | 2 +- 5 files changed, 69 insertions(+), 49 deletions(-) diff --git a/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java b/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java index 28e6402e..9aa284dd 100644 --- a/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java +++ b/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java @@ -27,10 +27,14 @@ ******************************************************************************/ package com.salesforce.hbase.index.util; +import java.io.IOException; +import java.util.Collection; import java.util.List; +import java.util.Map; import org.apache.hadoop.hbase.KeyValue; +import com.google.common.collect.Maps; import com.salesforce.hbase.index.ValueGetter; import com.salesforce.hbase.index.covered.data.LazyValueGetter; import com.salesforce.hbase.index.covered.update.ColumnReference; @@ -45,6 +49,19 @@ private IndexManagementUtil() { // private ctor for util classes } + public static ValueGetter createGetterFromKeyValues (Collection pendingUpdates) { + final Map valueMap = Maps.newHashMapWithExpectedSize(pendingUpdates.size()); + for (KeyValue kv : pendingUpdates) { + valueMap.put(new ColumnReference(kv.getFamily(), kv.getQualifier()), kv.getValue()); + } + return new ValueGetter() { + @Override + public byte[] getLatestValue(ColumnReference ref) throws IOException { + return valueMap.get(ref); + } + }; + + } public static ValueGetter createGetterFromScanner(Scanner scanner, byte[] currentRow) { return new LazyValueGetter(scanner, currentRow); } @@ -53,7 +70,7 @@ public static ValueGetter createGetterFromScanner(Scanner scanner, byte[] curren * This assumes that for any index, there are going to small number of columns, versus the number of * kvs in any one batch. */ - public static boolean updateMatchesColumns(List update, + public static boolean updateMatchesColumns(Collection update, List columns) { // check to see if the kvs in the new update even match any of the columns requested // assuming that for any index, there are going to small number of columns, versus the number of @@ -81,7 +98,7 @@ public static boolean updateMatchesColumns(List update, * iteration logic to search columns before kvs. */ public static boolean columnMatchesUpdate( - List columns, List update) { + List columns, Collection update) { boolean matches = false; outer: for (ColumnReference ref : columns) { for (KeyValue kv : update) { diff --git a/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java b/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java index a665e370..a6dfa72c 100644 --- a/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java +++ b/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java @@ -7,21 +7,25 @@ import java.io.DataOutputStream; import java.io.IOException; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Set; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.salesforce.hbase.index.ValueGetter; import com.salesforce.hbase.index.covered.update.ColumnReference; +import com.salesforce.hbase.index.util.IndexManagementUtil; import com.salesforce.phoenix.query.QueryConstants; import com.salesforce.phoenix.schema.ColumnModifier; import com.salesforce.phoenix.schema.PColumn; @@ -293,69 +297,68 @@ public byte[] buildRowKey(ValueGetter valueGetter, ImmutableBytesWritable rowKey } } - // TODO: remove once Jesse handles a Put and Delete on the same row @SuppressWarnings("deprecation") - public Put buildUpdateMutation(ValueGetter valueGetter, ImmutableBytesWritable dataRowKeyPtr) throws IOException { + public Put buildUpdateMutation(ValueGetter valueGetter, ImmutableBytesWritable dataRowKeyPtr, long ts) throws IOException { byte[] indexRowKey = this.buildRowKey(valueGetter, dataRowKeyPtr); Put put = new Put(indexRowKey); for (int i = 0; i < this.getCoverededColumns().size(); i++) { ColumnReference ref = this.getCoverededColumns().get(i); - byte[] iq = this.indexQualifiers.get(i); + byte[] cq = this.indexQualifiers.get(i); byte[] value = valueGetter.getLatestValue(ref); - if (value == null) { - // TODO: we should use a Delete here, but Jesse's framework can't handle that yet. - // This will work, but will cause an otherwise sparse index to be bloated with empty - // values for any unset covered columns. - put.add(ref.getFamily(), iq, ByteUtil.EMPTY_BYTE_ARRAY); - } else { - put.add(ref.getFamily(), iq, value); - } + put.add(ref.getFamily(), cq, ts, value); } // Add the empty key value - put.add(this.getEmptyKeyValueFamily(), QueryConstants.EMPTY_COLUMN_BYTES, ByteUtil.EMPTY_BYTE_ARRAY); + put.add(this.getEmptyKeyValueFamily(), QueryConstants.EMPTY_COLUMN_BYTES, ts, ByteUtil.EMPTY_BYTE_ARRAY); + // TODO: Jesse thinks I should remove this put.setWriteToWAL(false); return put; } - public Pair buildUpdateMutations(ValueGetter valueGetter, ImmutableBytesWritable dataRowKeyPtr) throws IOException { + public Put buildUpdateMutation(ValueGetter valueGetter, ImmutableBytesWritable dataRowKeyPtr) throws IOException { return buildUpdateMutation(valueGetter, dataRowKeyPtr, HConstants.LATEST_TIMESTAMP); } - @SuppressWarnings("deprecation") - public Pair buildUpdateMutation(ValueGetter valueGetter, ImmutableBytesWritable dataRowKeyPtr, long ts) throws IOException { - byte[] indexRowKey = this.buildRowKey(valueGetter, dataRowKeyPtr); - Delete delete = null; - Put put = new Put(indexRowKey); - for (int i = 0; i < this.getCoverededColumns().size(); i++) { - ColumnReference ref = this.getCoverededColumns().get(i); - byte[] iq = this.indexQualifiers.get(i); - byte[] value = valueGetter.getLatestValue(ref); - if (value == null) { - if (delete == null) { - delete = new Delete(indexRowKey); - delete.setWriteToWAL(false); + public Delete buildDeleteMutation(ValueGetter valueGetter, ImmutableBytesWritable dataRowKeyPtr, Collection pendingUpdates) throws IOException { + return buildDeleteMutation(valueGetter, dataRowKeyPtr, pendingUpdates, HConstants.LATEST_TIMESTAMP); + } + + private boolean indexedColumnsChanged(ValueGetter oldState, ValueGetter newState) throws IOException { + for (int i = 0; i < indexedColumns.size(); i++) { + ColumnReference ref = indexedColumns.get(i); + byte[] newValue = newState.getLatestValue(ref); + if (newValue != null) { // Indexed column was changed + byte[] oldValue = oldState.getLatestValue(ref); + if (oldValue == null || !Arrays.equals(newValue, oldValue)) { + return true; } - delete.deleteColumns(ref.getFamily(), iq, ts); - } else { - put.add(ref.getFamily(), iq, ts, value); } } - // Add the empty key value - put.add(this.getEmptyKeyValueFamily(), QueryConstants.EMPTY_COLUMN_BYTES, ts, ByteUtil.EMPTY_BYTE_ARRAY); - put.setWriteToWAL(false); - return new Pair(put,delete); - } - - public Delete buildDeleteMutation(ValueGetter valueGetter, ImmutableBytesWritable dataRowKeyPtr) throws IOException { - return buildDeleteMutation(valueGetter, dataRowKeyPtr, HConstants.LATEST_TIMESTAMP); + return false; } @SuppressWarnings("deprecation") - public Delete buildDeleteMutation(ValueGetter valueGetter, ImmutableBytesWritable dataRowKeyPtr, long ts) throws IOException { - byte[] indexRowKey = this.buildRowKey(valueGetter, dataRowKeyPtr); - Delete delete = new Delete(indexRowKey, ts, null); - delete.setWriteToWAL(false); - return delete; + public Delete buildDeleteMutation(ValueGetter oldState, ImmutableBytesWritable dataRowKeyPtr, Collection pendingUpdates, long ts) throws IOException { + byte[] indexRowKey = this.buildRowKey(oldState, dataRowKeyPtr); + // Delete the entire row if any of the indexed columns changed + if (pendingUpdates.isEmpty() || indexedColumnsChanged(oldState, IndexManagementUtil.createGetterFromKeyValues(pendingUpdates))) { // Deleting the entire row + Delete delete = new Delete(indexRowKey, ts, null); + delete.setWriteToWAL(false); + return delete; + } else { // Delete columns for missing key values + Delete delete = new Delete(indexRowKey); + Set pendingRefs = Sets.newHashSetWithExpectedSize(pendingUpdates.size()); + for (KeyValue kv : pendingUpdates) { + pendingRefs.add(new ColumnReference(kv.getFamily(), kv.getQualifier())); + } + for (int i = 0; i < this.getCoverededColumns().size(); i++) { + ColumnReference ref = this.getCoverededColumns().get(i); + if (oldState.getLatestValue(ref) != null && !pendingRefs.contains(ref)) { + byte[] cq = this.indexQualifiers.get(i); + delete.deleteColumns(ref.getFamily(), cq, ts); + } + } + return delete; + } } public byte[] getIndexTableName() { diff --git a/src/main/java/com/salesforce/phoenix/index/IndexMetaDataCacheClient.java b/src/main/java/com/salesforce/phoenix/index/IndexMetaDataCacheClient.java index 428288b9..64feee48 100644 --- a/src/main/java/com/salesforce/phoenix/index/IndexMetaDataCacheClient.java +++ b/src/main/java/com/salesforce/phoenix/index/IndexMetaDataCacheClient.java @@ -14,7 +14,7 @@ import com.salesforce.phoenix.util.ScanUtil; public class IndexMetaDataCacheClient { - private static final int USE_CACHE_THRESHOLD = 10; + private static final int USE_CACHE_THRESHOLD = 5; private final ServerCacheClient serverCache; diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java index 9c92a4fe..cf7faee0 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java @@ -135,13 +135,13 @@ public Iterable getIndexDeletes(TableState state) throws IOExceptio byte[] dataRowKey = state.getCurrentRowKey(); for (IndexMaintainer maintainer : indexMaintainers) { // TODO: if more efficient, I could do this just once with all columns in all indexes - Pair statePair = state.getIndexedColumnsTableState(maintainer.getIndexedColumns()); + Pair statePair = state.getIndexedColumnsTableState(maintainer.getAllColumns()); Scanner scanner = statePair.getFirst(); IndexUpdate indexUpdate = statePair.getSecond(); indexUpdate.setTable(maintainer.getIndexTableName()); ValueGetter valueGetter = IndexManagementUtil.createGetterFromScanner(scanner, dataRowKey); ptr.set(dataRowKey); - Delete delete = maintainer.buildDeleteMutation(valueGetter, ptr); + Delete delete = maintainer.buildDeleteMutation(valueGetter, ptr, state.getPendingUpdate()); scanner.close(); indexUpdate.setUpdate(delete); indexUpdates.add(indexUpdate); diff --git a/src/main/java/com/salesforce/phoenix/util/IndexUtil.java b/src/main/java/com/salesforce/phoenix/util/IndexUtil.java index 83f9626e..088ecf41 100644 --- a/src/main/java/com/salesforce/phoenix/util/IndexUtil.java +++ b/src/main/java/com/salesforce/phoenix/util/IndexUtil.java @@ -167,7 +167,7 @@ public byte[] getLatestValue(ColumnReference ref) { long ts = MetaDataUtil.getClientTimeStamp(dataMutation); ptr.set(dataMutation.getRow()); try { - indexMutations.add(maintainer.buildUpdateMutation(valueGetter, ptr, ts).getFirst()); + indexMutations.add(maintainer.buildUpdateMutation(valueGetter, ptr, ts)); } catch (IOException e) { throw new SQLException(e); } From 2ef225d4bb42010dd89c74e5ab39f90970c3f061 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Mon, 23 Sep 2013 17:20:08 -0700 Subject: [PATCH 065/102] Test covered column delete, add, modify for mutable indexes --- .../end2end/index/MutableIndexTest.java | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java index 13c7f679..1e567980 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java @@ -292,6 +292,99 @@ public void testSelectCF() throws Exception { assertFalse(rs.next()); } + @Test + public void testCoveredColumns() throws Exception { + String query; + ResultSet rs; + + Properties props = new Properties(TEST_PROPERTIES); + Connection conn = DriverManager.getConnection(getUrl(), props); + conn.setAutoCommit(false); + conn.createStatement().execute("CREATE TABLE " + DATA_TABLE_FULL_NAME + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)"); + query = "SELECT * FROM " + DATA_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertFalse(rs.next()); + + conn.createStatement().execute("CREATE INDEX " + INDEX_TABLE_NAME + " ON " + DATA_TABLE_FULL_NAME + " (v1) INCLUDE (v2)"); + query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertFalse(rs.next()); + + PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?)"); + stmt.setString(1,"a"); + stmt.setString(2, "x"); + stmt.setString(3, "1"); + stmt.execute(); + conn.commit(); + + query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("x",rs.getString(1)); + assertEquals("a",rs.getString(2)); + assertEquals("1",rs.getString(3)); + assertFalse(rs.next()); + + stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + "(k,v2) VALUES(?,?)"); + stmt.setString(1,"a"); + stmt.setString(2, null); + stmt.execute(); + conn.commit(); + + query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("x",rs.getString(1)); + assertEquals("a",rs.getString(2)); + assertNull(rs.getString(3)); + assertFalse(rs.next()); + + query = "SELECT * FROM " + DATA_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery("EXPLAIN " + query); + assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + INDEX_TABLE_FULL_NAME, QueryUtil.getExplainPlan(rs)); + + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("a",rs.getString(1)); + assertEquals("x",rs.getString(2)); + assertNull(rs.getString(3)); + assertFalse(rs.next()); + + stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + "(k,v2) VALUES(?,?)"); + stmt.setString(1,"a"); + stmt.setString(2,"3"); + stmt.execute(); + conn.commit(); + + query = "SELECT * FROM " + DATA_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery("EXPLAIN " + query); + assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + INDEX_TABLE_FULL_NAME, QueryUtil.getExplainPlan(rs)); + + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("a",rs.getString(1)); + assertEquals("x",rs.getString(2)); + assertEquals("3",rs.getString(3)); + assertFalse(rs.next()); + + stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + "(k,v2) VALUES(?,?)"); + stmt.setString(1,"a"); + stmt.setString(2,"4"); + stmt.execute(); + conn.commit(); + + query = "SELECT * FROM " + DATA_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery("EXPLAIN " + query); + assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + INDEX_TABLE_FULL_NAME, QueryUtil.getExplainPlan(rs)); + + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("a",rs.getString(1)); + assertEquals("x",rs.getString(2)); + assertEquals("4",rs.getString(3)); + assertFalse(rs.next()); + } + // @Test Broken, but Jesse is fixing public void testCompoundIndexKey() throws Exception { String query; From d7279577ae7ca05ab4f356443eec890a35f0e664 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Mon, 23 Sep 2013 18:56:46 -0700 Subject: [PATCH 066/102] Pad CHAR with space instead of null byte as per SQL spec --- .../expression/ComparisonExpression.java | 6 +- .../phoenix/expression/LiteralExpression.java | 4 +- .../salesforce/phoenix/schema/PDataType.java | 2 +- .../salesforce/phoenix/schema/PTableImpl.java | 5 +- .../salesforce/phoenix/util/SchemaUtil.java | 17 ------ .../salesforce/phoenix/util/StringUtil.java | 25 ++++---- .../compile/WhereClauseFilterTest.java | 7 ++- .../phoenix/end2end/OrderByTest.java | 57 ++++++++++++------- 8 files changed, 63 insertions(+), 60 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/expression/ComparisonExpression.java b/src/main/java/com/salesforce/phoenix/expression/ComparisonExpression.java index 3d15a771..4c60fe6d 100644 --- a/src/main/java/com/salesforce/phoenix/expression/ComparisonExpression.java +++ b/src/main/java/com/salesforce/phoenix/expression/ComparisonExpression.java @@ -39,7 +39,7 @@ import com.salesforce.phoenix.schema.PDataType; import com.salesforce.phoenix.schema.tuple.Tuple; import com.salesforce.phoenix.util.ByteUtil; -import com.salesforce.phoenix.util.SchemaUtil; +import com.salesforce.phoenix.util.StringUtil; /** @@ -115,10 +115,10 @@ public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { PDataType rhsDataType = children.get(1).getDataType(); ColumnModifier rhsColumnModifier = children.get(1).getColumnModifier(); if (rhsDataType == PDataType.CHAR) { - rhsLength = SchemaUtil.getUnpaddedCharLength(rhsBytes, rhsOffset, rhsLength, rhsColumnModifier); + rhsLength = StringUtil.getUnpaddedCharLength(rhsBytes, rhsOffset, rhsLength, rhsColumnModifier); } if (lhsDataType == PDataType.CHAR) { - lhsLength = SchemaUtil.getUnpaddedCharLength(lhsBytes, lhsOffset, lhsLength, lhsColumnModifier); + lhsLength = StringUtil.getUnpaddedCharLength(lhsBytes, lhsOffset, lhsLength, lhsColumnModifier); } int comparisonResult = lhsDataType.compareTo(lhsBytes, lhsOffset, lhsLength, lhsColumnModifier, diff --git a/src/main/java/com/salesforce/phoenix/expression/LiteralExpression.java b/src/main/java/com/salesforce/phoenix/expression/LiteralExpression.java index 24788353..aa368514 100644 --- a/src/main/java/com/salesforce/phoenix/expression/LiteralExpression.java +++ b/src/main/java/com/salesforce/phoenix/expression/LiteralExpression.java @@ -40,7 +40,7 @@ import com.salesforce.phoenix.schema.*; import com.salesforce.phoenix.schema.tuple.Tuple; import com.salesforce.phoenix.util.ByteUtil; -import com.salesforce.phoenix.util.SchemaUtil; +import com.salesforce.phoenix.util.StringUtil; @@ -125,7 +125,7 @@ public static LiteralExpression newConstant(Object value, PDataType type, Intege byte[] b = type.toBytes(value, columnModifier); if (type == PDataType.VARCHAR || type == PDataType.CHAR) { if (type == PDataType.CHAR && maxLength != null && b.length < maxLength) { - b = SchemaUtil.padChar(b, maxLength); + b = StringUtil.padChar(b, maxLength); } else if (value != null) { maxLength = ((String)value).length(); } diff --git a/src/main/java/com/salesforce/phoenix/schema/PDataType.java b/src/main/java/com/salesforce/phoenix/schema/PDataType.java index 8aa2006c..a8b866c2 100644 --- a/src/main/java/com/salesforce/phoenix/schema/PDataType.java +++ b/src/main/java/com/salesforce/phoenix/schema/PDataType.java @@ -199,7 +199,7 @@ public Object toObject(byte[] bytes, int offset, int length, PDataType actualTyp if (length == 0) { return null; } - length = SchemaUtil.getUnpaddedCharLength(bytes, offset, length, null); + length = StringUtil.getUnpaddedCharLength(bytes, offset, length, null); String s = Bytes.toString(bytes, offset, length); if (length != s.length()) { throw new IllegalDataException("CHAR types may only contain single byte characters (" + s + ")"); diff --git a/src/main/java/com/salesforce/phoenix/schema/PTableImpl.java b/src/main/java/com/salesforce/phoenix/schema/PTableImpl.java index d5586bb0..1ab5c510 100644 --- a/src/main/java/com/salesforce/phoenix/schema/PTableImpl.java +++ b/src/main/java/com/salesforce/phoenix/schema/PTableImpl.java @@ -65,6 +65,7 @@ import com.salesforce.phoenix.schema.stat.PTableStatsImpl; import com.salesforce.phoenix.util.ByteUtil; import com.salesforce.phoenix.util.SchemaUtil; +import com.salesforce.phoenix.util.StringUtil; import com.salesforce.phoenix.util.TrustedByteArrayOutputStream; @@ -353,7 +354,7 @@ public int newKey(ImmutableBytesWritable key, byte[][] values) { } Integer byteSize = column.getByteSize(); if (type.isFixedWidth() && byteValue.length <= byteSize) { - byteValue = SchemaUtil.padChar(byteValue, byteSize); + byteValue = StringUtil.padChar(byteValue, byteSize); } else if (byteSize != null && byteValue.length > byteSize) { throw new ConstraintViolationException(name.getString() + "." + column.getName().getString() + " may not exceed " + byteSize + " bytes (" + SchemaUtil.toString(type, byteValue) + ")"); } @@ -517,7 +518,7 @@ public void setValue(PColumn column, byte[] byteValue) { } else { Integer byteSize = column.getByteSize(); if (type.isFixedWidth() && byteValue.length <= byteSize) { - byteValue = SchemaUtil.padChar(byteValue, byteSize); + byteValue = StringUtil.padChar(byteValue, byteSize); } else if (byteSize != null && byteValue.length > byteSize) { throw new ConstraintViolationException(name.getString() + "." + column.getName().getString() + " may not exceed " + byteSize + " bytes (" + type.toObject(byteValue) + ")"); } diff --git a/src/main/java/com/salesforce/phoenix/util/SchemaUtil.java b/src/main/java/com/salesforce/phoenix/util/SchemaUtil.java index 06b1ea2b..76a038a1 100644 --- a/src/main/java/com/salesforce/phoenix/util/SchemaUtil.java +++ b/src/main/java/com/salesforce/phoenix/util/SchemaUtil.java @@ -50,7 +50,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -89,7 +88,6 @@ import com.salesforce.phoenix.query.QueryConstants; import com.salesforce.phoenix.schema.AmbiguousColumnException; import com.salesforce.phoenix.schema.ColumnFamilyNotFoundException; -import com.salesforce.phoenix.schema.ColumnModifier; import com.salesforce.phoenix.schema.ColumnNotFoundException; import com.salesforce.phoenix.schema.PColumn; import com.salesforce.phoenix.schema.PColumnFamily; @@ -112,7 +110,6 @@ public class SchemaUtil { private static final Logger logger = LoggerFactory.getLogger(SchemaUtil.class); private static final int VAR_LENGTH_ESTIMATE = 10; - private static final byte PAD_BYTE = (byte)0; public static final DataBlockEncoding DEFAULT_DATA_BLOCK_ENCODING = DataBlockEncoding.FAST_DIFF; /** @@ -318,16 +315,6 @@ public static String getName(byte[] nameOne, byte[] nameTwo) { return Bytes.toString(getNameAsBytes(nameOne,nameTwo)); } - public static int getUnpaddedCharLength(byte[] b, int offset, int length, ColumnModifier columnModifier) { - int i = offset + length -1; - // If bytes are inverted, we need to invert the byte we're looking for too - byte padByte = columnModifier == null ? PAD_BYTE : columnModifier.apply(PAD_BYTE); - while(i > offset && b[i] == padByte) { - i--; - } - return i - offset + 1; - } - public static int getVarCharLength(byte[] buf, int keyOffset, int maxLength) { return getVarCharLength(buf, keyOffset, maxLength, 1); } @@ -418,10 +405,6 @@ public static boolean isMetaTable(byte[] tableName) { return Bytes.compareTo(tableName, TYPE_TABLE_NAME_BYTES) == 0; } - public static byte[] padChar(byte[] byteValue, Integer byteSize) { - return Arrays.copyOf(byteValue, byteSize); - } - public static boolean isMetaTable(String schemaName, String tableName) { return PhoenixDatabaseMetaData.TYPE_SCHEMA.equals(schemaName) && PhoenixDatabaseMetaData.TYPE_TABLE.equals(tableName); } diff --git a/src/main/java/com/salesforce/phoenix/util/StringUtil.java b/src/main/java/com/salesforce/phoenix/util/StringUtil.java index 7331e0d1..612383da 100644 --- a/src/main/java/com/salesforce/phoenix/util/StringUtil.java +++ b/src/main/java/com/salesforce/phoenix/util/StringUtil.java @@ -28,6 +28,7 @@ package com.salesforce.phoenix.util; import java.io.UnsupportedEncodingException; +import java.util.Arrays; import org.apache.hadoop.hbase.util.Bytes; @@ -185,15 +186,10 @@ public static int getFirstNonBlankCharIdxFromStart(byte[] string, int offset, in if (string[i] != space) { break; } -// if ((getBytesInChar(string[i]) != 1 || -// (getBytesInChar(string[i]) == 1 && SPACE_UTF8 < string[i] && string[i] != 0x7f))) { -// break; -// } } return i; } - //Why is this so complicated? Can't we just test the byte against SPACE_UTF8? public static int getFirstNonBlankCharIdxFromEnd(byte[] string, int offset, int length, ColumnModifier columnModifier) { int i = offset + length - 1; byte space = columnModifier == null ? SPACE_UTF8 : MOD_SPACE_UTF8[columnModifier.ordinal()]; @@ -201,12 +197,7 @@ public static int getFirstNonBlankCharIdxFromEnd(byte[] string, int offset, int if (string[i] != space) { break; } -// int b = string[i] & 0xff; -// if (((b & BYTES_1_MASK) != 0) || -// ((b & BYTES_1_MASK) == 0 && SPACE_UTF8 < b && b != 0x7f)) { -// break; -// } - } + } return i; } @@ -222,4 +213,16 @@ public static byte[] toBytes(String input) { public static String escapeLike(String s) { return replace(s, LIKE_UNESCAPED_SEQS, LIKE_ESCAPE_SEQS); } + + public static int getUnpaddedCharLength(byte[] b, int offset, int length, ColumnModifier columnModifier) { + return getFirstNonBlankCharIdxFromEnd(b, offset, length, columnModifier) - offset + 1; + } + + public static byte[] padChar(byte[] value, Integer byteSize) { + byte[] newValue = Arrays.copyOf(value, byteSize); + if (newValue.length > value.length) { + Arrays.fill(newValue, value.length, newValue.length, SPACE_UTF8); + } + return newValue; + } } diff --git a/src/test/java/com/salesforce/phoenix/compile/WhereClauseFilterTest.java b/src/test/java/com/salesforce/phoenix/compile/WhereClauseFilterTest.java index b96b4e19..678edf62 100644 --- a/src/test/java/com/salesforce/phoenix/compile/WhereClauseFilterTest.java +++ b/src/test/java/com/salesforce/phoenix/compile/WhereClauseFilterTest.java @@ -82,6 +82,7 @@ import com.salesforce.phoenix.util.ByteUtil; import com.salesforce.phoenix.util.DateUtil; import com.salesforce.phoenix.util.NumberUtil; +import com.salesforce.phoenix.util.StringUtil; public class WhereClauseFilterTest extends BaseConnectionlessQueryTest { @@ -350,7 +351,7 @@ public void testPaddedStartStopKey() throws SQLException { ColumnResolver resolver = FromCompiler.getResolver(statement, pconn); StatementContext context = new StatementContext(statement, pconn, resolver, binds, scan); statement = compileStatement(context, statement, resolver, binds, scan, 2, null); - assertArrayEquals(ByteUtil.concat(Bytes.toBytes(tenantId), Bytes.toBytes(keyPrefix), new byte[15-keyPrefix.length()]),scan.getStartRow()); + assertArrayEquals(ByteUtil.concat(Bytes.toBytes(tenantId), StringUtil.padChar(Bytes.toBytes(keyPrefix), 15)),scan.getStartRow()); assertArrayEquals(ByteUtil.nextKey(scan.getStartRow()),scan.getStopRow()); } @@ -834,8 +835,8 @@ public void testPartialRangeFilter() throws SQLException { statement = compileStatement(context, statement, resolver, binds, scan, 2, null); assertNull(scan.getFilter()); - byte[] wideLower = ByteUtil.nextKey(ByteUtil.fillKey(Bytes.toBytes(tenantId1), 15)); - byte[] wideUpper = ByteUtil.fillKey(Bytes.toBytes(tenantId2), 15); + byte[] wideLower = ByteUtil.nextKey(StringUtil.padChar(Bytes.toBytes(tenantId1), 15)); + byte[] wideUpper = StringUtil.padChar(Bytes.toBytes(tenantId2), 15); assertArrayEquals(wideLower, scan.getStartRow()); assertArrayEquals(wideUpper, scan.getStopRow()); } diff --git a/src/test/java/com/salesforce/phoenix/end2end/OrderByTest.java b/src/test/java/com/salesforce/phoenix/end2end/OrderByTest.java index b591461c..9ce9b351 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/OrderByTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/OrderByTest.java @@ -27,10 +27,25 @@ ******************************************************************************/ package com.salesforce.phoenix.end2end; -import static com.salesforce.phoenix.util.TestUtil.*; -import static org.junit.Assert.*; +import static com.salesforce.phoenix.util.TestUtil.PHOENIX_JDBC_URL; +import static com.salesforce.phoenix.util.TestUtil.ROW1; +import static com.salesforce.phoenix.util.TestUtil.ROW2; +import static com.salesforce.phoenix.util.TestUtil.ROW3; +import static com.salesforce.phoenix.util.TestUtil.ROW4; +import static com.salesforce.phoenix.util.TestUtil.ROW5; +import static com.salesforce.phoenix.util.TestUtil.ROW6; +import static com.salesforce.phoenix.util.TestUtil.ROW7; +import static com.salesforce.phoenix.util.TestUtil.ROW8; +import static com.salesforce.phoenix.util.TestUtil.ROW9; +import static com.salesforce.phoenix.util.TestUtil.TEST_PROPERTIES; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; -import java.sql.*; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.util.Properties; import org.junit.Test; @@ -52,23 +67,23 @@ public void testMultiOrderByExpr() throws Exception { PreparedStatement statement = conn.prepareStatement(query); ResultSet rs = statement.executeQuery(); assertTrue (rs.next()); - assertEquals(rs.getString(1), ROW1); + assertEquals(ROW1,rs.getString(1)); assertTrue (rs.next()); - assertEquals(rs.getString(1), ROW4); + assertEquals(ROW4,rs.getString(1)); assertTrue (rs.next()); - assertEquals(rs.getString(1), ROW7); + assertEquals(ROW7,rs.getString(1)); assertTrue (rs.next()); - assertEquals(rs.getString(1), ROW2); + assertEquals(ROW2,rs.getString(1)); assertTrue (rs.next()); - assertEquals(rs.getString(1), ROW5); + assertEquals(ROW5,rs.getString(1)); assertTrue (rs.next()); - assertEquals(rs.getString(1), ROW8); + assertEquals(ROW8,rs.getString(1)); assertTrue (rs.next()); - assertEquals(rs.getString(1), ROW3); + assertEquals(ROW3,rs.getString(1)); assertTrue (rs.next()); - assertEquals(rs.getString(1), ROW6); + assertEquals(ROW6,rs.getString(1)); assertTrue (rs.next()); - assertEquals(rs.getString(1), ROW9); + assertEquals(ROW9,rs.getString(1)); assertFalse(rs.next()); } finally { @@ -90,23 +105,23 @@ public void testDescMultiOrderByExpr() throws Exception { PreparedStatement statement = conn.prepareStatement(query); ResultSet rs = statement.executeQuery(); assertTrue (rs.next()); - assertEquals(rs.getString(1), ROW9); + assertEquals(ROW9,rs.getString(1)); assertTrue (rs.next()); - assertEquals(rs.getString(1), ROW6); + assertEquals(ROW6,rs.getString(1)); assertTrue (rs.next()); - assertEquals(rs.getString(1), ROW3); + assertEquals(ROW3,rs.getString(1)); assertTrue (rs.next()); - assertEquals(rs.getString(1), ROW8); + assertEquals(ROW8,rs.getString(1)); assertTrue (rs.next()); - assertEquals(rs.getString(1), ROW5); + assertEquals(ROW5,rs.getString(1)); assertTrue (rs.next()); - assertEquals(rs.getString(1), ROW2); + assertEquals(ROW2,rs.getString(1)); assertTrue (rs.next()); - assertEquals(rs.getString(1), ROW7); + assertEquals(ROW7,rs.getString(1)); assertTrue (rs.next()); - assertEquals(rs.getString(1), ROW4); + assertEquals(ROW4,rs.getString(1)); assertTrue (rs.next()); - assertEquals(rs.getString(1), ROW1); + assertEquals(ROW1,rs.getString(1)); assertFalse(rs.next()); } finally { From 5160dae85724a2fcd17fd84163652dd357a25807 Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Mon, 23 Sep 2013 16:08:03 -0700 Subject: [PATCH 067/102] Enabling batch support for indexes + bug fixes. Now we correctly handle batches of updates - all updates for a given row are grouped into a single Index Batch which is then processed by the usual update mechanisms. Other Bug Fixes: - Wrong ordering in MutableIndexTest - Wasn't setting pending updates before cleanup - Adding debugging statements (and toString impls) - Removing ColumnFamilyIndexer since its a worse example than the CoveredColumn or Phoenix indexer - making stopped AtmoicBoolean in IndexWriter to help race conditions in stop/write -- might be cause of the concurrent modification issue when HRegion#doClose seen sometimes in the test. --- pom.xml | 2 +- .../hbase/index/IndexLogRollSynchronizer.java | 8 +- .../salesforce/hbase/index/IndexWriter.java | 17 +- .../com/salesforce/hbase/index/Indexer.java | 224 +++++++++++++++--- .../hbase/index/builder/BaseIndexBuilder.java | 12 + .../hbase/index/builder/IndexBuilder.java | 24 +- .../builder/example/ColumnFamilyIndexer.java | 217 ----------------- .../covered/CoveredColumnsIndexBuilder.java | 53 +++-- .../hbase/index/covered/IndexCodec.java | 21 +- .../index/covered/data/IndexMemStore.java | 23 ++ .../index/covered/example/CoveredColumn.java | 36 +-- .../example/CoveredColumnIndexCodec.java | 4 +- .../covered/update/IndexUpdateManager.java | 18 +- .../coprocessor/MetaDataEndpointImpl.java | 7 +- .../phoenix/index/BaseIndexCodec.java | 68 ++++++ .../phoenix/index/IndexMaintainer.java | 40 ++-- .../phoenix/index/PhoenixIndexBuilder.java | 9 +- .../phoenix/index/PhoenixIndexCodec.java | 54 ++--- .../phoenix/jdbc/PhoenixResultSet.java | 7 +- .../query/ConnectionQueryServicesImpl.java | 6 +- .../phoenix/schema/tuple/ResultTuple.java | 20 +- .../hbase/index/TestEndtoEndIndexing.java | 207 ---------------- .../TestFailForUnsupportedHBaseVersions.java | 20 +- .../hbase/index/TestIndexWriter.java | 2 - .../covered/CoveredIndexCodecForTesting.java | 4 +- ...estEndToEndCoveredColumnsIndexBuilder.java | 42 ++-- .../example/TestEndToEndCoveredIndexing.java | 2 +- .../TestEndtoEndIndexingWithCompression.java | 9 +- .../end2end/index/MutableIndexTest.java | 98 +++++++- .../wal/TestWALReplayWithIndexWrites.java | 31 ++- 30 files changed, 647 insertions(+), 638 deletions(-) delete mode 100644 src/main/java/com/salesforce/hbase/index/builder/example/ColumnFamilyIndexer.java create mode 100644 src/main/java/com/salesforce/phoenix/index/BaseIndexCodec.java delete mode 100644 src/test/java/com/salesforce/hbase/index/TestEndtoEndIndexing.java rename src/test/java/com/salesforce/hbase/index/{ => covered/example}/TestEndtoEndIndexingWithCompression.java (86%) diff --git a/pom.xml b/pom.xml index f6d2218d..00e39b06 100644 --- a/pom.xml +++ b/pom.xml @@ -313,7 +313,7 @@ maven-surefire-plugin 2.13 - -Xmx2500m + -enableassertions -Xmx2500m -Djava.security.egd=file:/dev/./urandom ${test.output.tofile} diff --git a/src/main/java/com/salesforce/hbase/index/IndexLogRollSynchronizer.java b/src/main/java/com/salesforce/hbase/index/IndexLogRollSynchronizer.java index bc9001c3..c048f84f 100644 --- a/src/main/java/com/salesforce/hbase/index/IndexLogRollSynchronizer.java +++ b/src/main/java/com/salesforce/hbase/index/IndexLogRollSynchronizer.java @@ -30,6 +30,8 @@ import java.io.IOException; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; @@ -75,6 +77,7 @@ */ public class IndexLogRollSynchronizer implements WALActionsListener { + private static final Log LOG = LogFactory.getLog(IndexLogRollSynchronizer.class); private WriteLock logArchiveLock; public IndexLogRollSynchronizer(WriteLock logWriteLock){ @@ -85,18 +88,21 @@ public IndexLogRollSynchronizer(WriteLock logWriteLock){ @Override public void preLogArchive(Path oldPath, Path newPath) throws IOException { //take a write lock on the index - any pending index updates will complete before we finish + LOG.debug("Taking INDEX_UPDATE writelock"); logArchiveLock.lock(); } @Override public void postLogArchive(Path oldPath, Path newPath) throws IOException { // done archiving the logs, any WAL updates will be replayed on failure + LOG.debug("Releasing INDEX_UPDATE writelock"); logArchiveLock.unlock(); } @Override public void logCloseRequested() { - // don't care- before this is called, all the HRegions are closed, so we can't get any new requests and all pending request can finish before the WAL closes. + // don't care- before this is called, all the HRegions are closed, so we can't get any new + // requests and all pending request can finish before the WAL closes. } @Override diff --git a/src/main/java/com/salesforce/hbase/index/IndexWriter.java b/src/main/java/com/salesforce/hbase/index/IndexWriter.java index f3dc3196..22aefb0f 100644 --- a/src/main/java/com/salesforce/hbase/index/IndexWriter.java +++ b/src/main/java/com/salesforce/hbase/index/IndexWriter.java @@ -44,6 +44,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -88,7 +89,7 @@ public class IndexWriter implements Stoppable { private final CapturingAbortable abortable; private final HTableFactory factory; private ListeningExecutorService writerPool; - private boolean stopped; + private AtomicBoolean stopped = new AtomicBoolean(false); /** * @param sourceInfo log info string about where we are writing from @@ -217,7 +218,7 @@ public void write(Collection> indexUpdates) final List mutations = (List) entry.getValue(); final HTableInterfaceReference tableReference = entry.getKey(); // early exit - no need to submit new tasks if we are shutting down - if (this.stopped || this.abortable.isAborted()) { + if (this.stopped.get() || this.abortable.isAborted()) { break; } @@ -263,7 +264,7 @@ public Void call() throws Exception { } private void throwFailureIfDone() throws CannotReachIndexException { - if (stopped || abortable.isAborted() || Thread.currentThread().isInterrupted()) { + if (stopped.get() || abortable.isAborted() || Thread.currentThread().isInterrupted()) { throw new CannotReachIndexException( "Pool closed, not attempting to write to the index!", null); } @@ -336,8 +337,8 @@ private void killYourself(Throwable cause) { try { this.abortable.abort(msg, cause); } catch (Exception e) { - LOG.fatal("Couldn't abort this server to preserve index writes, attempting to hard kill the server from" - + this.sourceInfo); + LOG.fatal("Couldn't abort this server to preserve index writes, " + + "attempting to hard kill the server from" + this.sourceInfo); System.exit(1); } } @@ -371,10 +372,10 @@ public static Multimap resolveTableReference @Override public void stop(String why) { - if (this.stopped) { + if (!this.stopped.compareAndSet(false, true)) { + // already stopped return; } - this.stopped = true; LOG.debug("Stopping because " + why); this.writerPool.shutdownNow(); this.factory.shutdown(); @@ -382,6 +383,6 @@ public void stop(String why) { @Override public boolean isStopped() { - return this.stopped; + return this.stopped.get(); } } \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/Indexer.java b/src/main/java/com/salesforce/hbase/index/Indexer.java index cf80649d..1d102392 100644 --- a/src/main/java/com/salesforce/hbase/index/Indexer.java +++ b/src/main/java/com/salesforce/hbase/index/Indexer.java @@ -27,6 +27,8 @@ ******************************************************************************/ package com.salesforce.hbase.index; +import java.io.DataInput; +import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -34,6 +36,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; @@ -63,6 +66,7 @@ import org.apache.hadoop.hbase.util.Pair; import com.salesforce.hbase.index.builder.IndexBuilder; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.hbase.index.wal.IndexedKeyValue; /** @@ -101,6 +105,14 @@ public class Indexer extends BaseRegionObserver { */ public static final String CHECK_VERSION_CONF_KEY = "com.saleforce.hbase.index.checkversion"; + /** + * Marker {@link KeyValue} to indicate that we are doing a batch operation. Needed because the + * coprocessor framework throws away the WALEdit from the prePut/preDelete hooks when checking a + * batch if there were no {@link KeyValue}s attached to the {@link WALEdit}. When you get down to + * the preBatch hook, there won't be any WALEdits to which to add the index updates. + */ + private static KeyValue BATCH_MARKER = new KeyValue(); + @Override public void start(CoprocessorEnvironment e) throws IOException { @@ -149,33 +161,175 @@ public void stop(CoprocessorEnvironment e) throws IOException { @Override public void prePut(final ObserverContext c, final Put put, final WALEdit edit, final boolean writeToWAL) throws IOException { - // get the mapping for index column -> target index table - if (!this.builder.isEnabled(put)) { - return; - } - Collection> indexUpdates = this.builder.getIndexUpdate(put); - - doPre(indexUpdates, edit, writeToWAL); + // just have to add a batch marker to the WALEdit so we get the edit again in the batch + // processing step + edit.add(BATCH_MARKER); } @Override public void preDelete(ObserverContext e, Delete delete, WALEdit edit, boolean writeToWAL) throws IOException { - // short circuit so we don't waste time + // short circuit so we don't waste time. if (!this.builder.isEnabled(delete)) { return; } + // if we are making the update as part of a batch, we need to add in a batch marker so the WAL + // is retained + else if (this.builder.getBatchId(delete) != null) { + edit.add(BATCH_MARKER); + return; + } + // get the mapping for index column -> target index table Collection> indexUpdates = this.builder.getIndexUpdate(delete); - doPre(indexUpdates, edit, writeToWAL); + if (doPre(indexUpdates, edit, writeToWAL)) { + // lock the log, so we are sure that index write gets atomically committed + LOG.debug("Taking INDEX_UPDATE readlock for delete"); + INDEX_UPDATE_LOCK.lock(); + } } - private void doPre(Collection> indexUpdates, - final WALEdit edit, final boolean writeToWAL) throws IOException { + @SuppressWarnings("deprecation") + @Override + public void preBatchMutate(ObserverContext c, + MiniBatchOperationInProgress> miniBatchOp) throws IOException { + + // first group all the updates for a single row into a single update to be processed + Map mutations = + new HashMap(); + for (int i = 0; i < miniBatchOp.size(); i++) { + // remove the batch keyvalue marker - its added for all puts + WALEdit edit = miniBatchOp.getWalEdit(i); + // we don't have a WALEdit for immutable index cases, which still see this path + // we could check is indexing is enable for the mutation in prePut and then just skip this + // after checking here, but this saves us the checking again. + if (edit != null) { + KeyValue kv = edit.getKeyValues().remove(0); + assert kv == BATCH_MARKER : "Didn't get the batch marker from the WALEdit during a batch"; + } + Pair op = miniBatchOp.getOperation(i); + Mutation m = op.getFirst(); + // skip this mutation if we aren't enabling indexing + if (!this.builder.isEnabled(m)) { + continue; + } + + // add the mutation to the batch set + ImmutableBytesPtr row = new ImmutableBytesPtr(m.getRow()); + MultiMutation stored = mutations.get(row); + // we haven't seen this row before, so add it + if (stored == null) { + stored = new MultiMutation(row, m.getWriteToWAL()); + mutations.put(row, stored); + } + stored.addAll(m); + } + + // only mention that we are indexing a batch after when know we have some edits + if (mutations.entrySet().size() > 0) { + this.builder.batchStarted(miniBatchOp); + } else { + return; + } + + // dump all the index updates into a single WAL. They will get combined in the end anyways, so + // don't worry which one we get + WALEdit edit = miniBatchOp.getWalEdit(0); + + // do the usual updates + boolean lock = false; + for (Entry entry : mutations.entrySet()) { + Mutation m = entry.getValue(); + Collection> indexUpdates = this.builder.getIndexUpdate(m); + + if (doPre(indexUpdates, edit, m.getWriteToWAL())) { + lock = true; + } + } + // if any of the updates in the batch updated the WAL, then we need to the lock the WAL + if (lock) { + LOG.debug("Taking INDEX_UPDATE readlock for batch mutation"); + INDEX_UPDATE_LOCK.lock(); + } + } + + private class MultiMutation extends Mutation { + + private ImmutableBytesPtr rowKey; + + public MultiMutation(ImmutableBytesPtr rowkey, boolean writeToWal) { + this.rowKey = rowkey; + this.writeToWAL = writeToWal; + } + + /** + * @param stored + */ + public void addAll(Mutation stored) { + // add all the kvs + for (Entry> kvs : stored.getFamilyMap().entrySet()) { + byte[] family = kvs.getKey(); + List list = getKeyValueList(family, kvs.getValue().size()); + list.addAll(kvs.getValue()); + familyMap.put(family, list); + } + + // add all the attributes, not overriding already stored ones + for (Entry attrib : stored.getAttributesMap().entrySet()) { + if (this.getAttribute(attrib.getKey()) == null) { + this.setAttribute(attrib.getKey(), attrib.getValue()); + } + } + if (stored.getWriteToWAL()) { + this.writeToWAL = true; + } + } + + private List getKeyValueList(byte[] family, int hint) { + List list = familyMap.get(family); + if (list == null) { + list = new ArrayList(hint); + } + return list; + } + + @Override + public byte[] getRow(){ + return this.rowKey.copyBytesIfNecessary(); + } + + @Override + public int hashCode() { + return this.rowKey.hashCode(); + } + + @Override + public boolean equals(Object o) { + return o == null ? false : o.hashCode() == this.hashCode(); + } + + @Override + public void readFields(DataInput arg0) throws IOException { + throw new UnsupportedOperationException("MultiMutations cannot be read/written"); + } + + @Override + public void write(DataOutput arg0) throws IOException { + throw new UnsupportedOperationException("MultiMutations cannot be read/written"); + } + } + + /** + * Add the index updates to the WAL, or write to the index table, if the WAL has been disabled + * @return true if the WAL has been updated. + * @throws IOException + */ + private boolean doPre(Collection> indexUpdates, final WALEdit edit, + final boolean writeToWAL) throws IOException { // no index updates, so we are done if (indexUpdates == null || indexUpdates.size() == 0) { - return; + return false; } // if writing to wal is disabled, we never see the WALEdit updates down the way, so do the index @@ -183,7 +337,7 @@ private void doPre(Collection> indexUpdates, if (!writeToWAL) { try { this.writer.write(indexUpdates); - return; + return false; } catch (CannotReachIndexException e) { LOG.error("Failed to update index with entries:" + indexUpdates, e); throw new IOException(e); @@ -195,20 +349,7 @@ private void doPre(Collection> indexUpdates, edit.add(new IndexedKeyValue(entry.getSecond(), entry.getFirst())); } - // lock the log, so we are sure that index write gets atomically committed - INDEX_UPDATE_LOCK.lock(); - } - - @Override - public void preBatchMutate(ObserverContext c, - MiniBatchOperationInProgress> miniBatchOp) throws IOException { - this.builder.batchStarted(miniBatchOp); - } - - @Override - public void postBatchMutate(ObserverContext c, - MiniBatchOperationInProgress> miniBatchOp) throws IOException { - this.builder.batchCompleted(miniBatchOp); + return true; } @Override @@ -223,23 +364,25 @@ public void postDelete(ObserverContext e, Delete d doPost(edit,delete, writeToWAL); } + @Override + public void postBatchMutate(ObserverContext c, + MiniBatchOperationInProgress> miniBatchOp) throws IOException { + this.builder.batchCompleted(miniBatchOp); + // noop for the rest of the indexer - its handled by the first call to put/delete + } + /** * @param edit * @param writeToWAL */ private void doPost(WALEdit edit, Mutation m, boolean writeToWAL) { - if (!writeToWAL) { + //short circuit, if we don't need to do any work + if (!writeToWAL || !this.builder.isEnabled(m)) { // already did the index update in prePut, so we are done return; } - // turns out, even doing the checking here for the index updates will cause a huge slowdown. Its - // up to the codec to be smart about how it manages this (and leak a little of the - // implementation here, but that's the way optimization go). - if(!this.builder.isEnabled(m)){ - return; - } - // there is a little bit of excess here- we iterate all the non-index kvs for this check first + // there is a little bit of excess here- we iterate all the non-indexed kvs for this check first // and then do it again later when getting out the index updates. This should be pretty minor // though, compared to the rest of the runtime IndexedKeyValue ikv = getFirstIndexedKeyValue(edit); @@ -268,11 +411,14 @@ private void doPost(WALEdit edit, Mutation m, boolean writeToWAL) { // mark the batch as having been written. In the single-update case, this never gets check // again, but in the batch case, we will check it again (see above). ikv.markBatchFinished(); + + // release the lock on the index, we wrote everything properly + // we took the lock for each Put/Delete, so we have to release it a matching number of times + // batch cases only take the lock once, so we need to make sure we don't over-release the + // lock. + LOG.debug("Releasing INDEX_UPDATE readlock"); + INDEX_UPDATE_LOCK.unlock(); } - - // release the lock on the index, we wrote everything properly - // we took the lock for each Put/Delete, so we have to release it a matching number of times - INDEX_UPDATE_LOCK.unlock(); } /** diff --git a/src/main/java/com/salesforce/hbase/index/builder/BaseIndexBuilder.java b/src/main/java/com/salesforce/hbase/index/builder/BaseIndexBuilder.java index 68cf1f7b..6570d03b 100644 --- a/src/main/java/com/salesforce/hbase/index/builder/BaseIndexBuilder.java +++ b/src/main/java/com/salesforce/hbase/index/builder/BaseIndexBuilder.java @@ -75,4 +75,16 @@ public void batchCompleted(MiniBatchOperationInProgress> public boolean isEnabled(Mutation m) { return true; } + + /** + * {@inheritDoc} + *

+ * By default, assumes that all mutations should not be batched. That is to say, each + * mutation always applies to different rows, even if they are in the same batch, or are + * independent updates. + */ + @Override + public byte[] getBatchId(Mutation m) { + return null; + } } \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/builder/IndexBuilder.java b/src/main/java/com/salesforce/hbase/index/builder/IndexBuilder.java index 4d10593b..07fc2f03 100644 --- a/src/main/java/com/salesforce/hbase/index/builder/IndexBuilder.java +++ b/src/main/java/com/salesforce/hbase/index/builder/IndexBuilder.java @@ -62,19 +62,24 @@ public interface IndexBuilder { public void setup(RegionCoprocessorEnvironment env) throws IOException; /** - * Your opportunity to update any/all index tables based on the delete of the primary table row. + * Your opportunity to update any/all index tables based on the update of the primary table row. * Its up to your implementation to ensure that timestamps match between the primary and index * tables. - * @param put {@link Put} to the primary table that may be indexed + *

+ * The mutation is a generic mutation (not a {@link Put} or a {@link Delete}), as it actually + * cooresponds to a batch update. Its important to note that {@link Put}s always go through the + * batch update code path, so a single {@link Put}, flushed to the server will come through here + * and update the primary table. + * @param mutation update to the primary table to be indexed. * @return a Map of the mutations to make -> target index table name * @throws IOException on failure */ - public Collection> getIndexUpdate(Put put) throws IOException; + public Collection> getIndexUpdate(Mutation mutation) throws IOException; /** - * The counter-part to {@link #getIndexUpdate(Put)} - your opportunity to update any/all index - * tables based on the delete of the primary table row. Its up to your implementation to ensure - * that timestamps match between the primary and index tables. + * The counter-part to {@link #getIndexUpdate(Mutation)} - your opportunity to update any/all + * index tables based on the delete of the primary table row. Its up to your implementation to + * ensure that timestamps match between the primary and index tables. * @param delete {@link Delete} to the primary table that may be indexed * @return a {@link Map} of the mutations to make -> target index table name * @throws IOException on failure @@ -123,4 +128,11 @@ public Collection> getIndexUpdateForFilteredRows( * basis, as each codec is instantiated per-region. */ public boolean isEnabled(Mutation m); + + /** + * @param m mutation that has been received by the indexer and is waiting to be indexed + * @return the ID of batch to which the Mutation belongs, or null if the mutation is not + * part of a batch. + */ + public byte[] getBatchId(Mutation m); } \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/builder/example/ColumnFamilyIndexer.java b/src/main/java/com/salesforce/hbase/index/builder/example/ColumnFamilyIndexer.java deleted file mode 100644 index da276a09..00000000 --- a/src/main/java/com/salesforce/hbase/index/builder/example/ColumnFamilyIndexer.java +++ /dev/null @@ -1,217 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2013, Salesforce.com, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of Salesforce.com nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ******************************************************************************/ -package com.salesforce.hbase.index.builder.example; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -import org.apache.commons.lang.ArrayUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.Mutation; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Pair; - -import com.salesforce.hbase.index.Indexer; -import com.salesforce.hbase.index.builder.BaseIndexBuilder; -import com.salesforce.hbase.index.util.ImmutableBytesPtr; - -/** - * Simple indexer that just indexes rows based on their column families - *

- * Doesn't do any index cleanup; this is just a basic example case. - */ -public class ColumnFamilyIndexer extends BaseIndexBuilder { - - private static final String INDEX_TO_TABLE_CONF_PREFX = "hbase.index.family."; - private static final String INDEX_TO_TABLE_COUNT_KEY = INDEX_TO_TABLE_CONF_PREFX + "families"; - private static final String SEPARATOR = ","; - - static final byte[] INDEX_ROW_COLUMN_FAMILY = Bytes.toBytes("ROW"); - static final byte[] INDEX_REMAINING_COLUMN_FAMILY = Bytes.toBytes("REMAINING"); - - public static void enableIndexing(HTableDescriptor desc, Map familyMap) - throws IOException { - // not indexing any families, so we shouldn't add the indexer - if (familyMap == null || familyMap.size() == 0) { - return; - } - Map opts = new HashMap(); - List families = new ArrayList(familyMap.size()); - - for (Entry family : familyMap.entrySet()) { - String fam = Bytes.toString(family.getKey()); - opts.put(INDEX_TO_TABLE_CONF_PREFX + fam, family.getValue()); - families.add(fam); - } - - // add the list of families so we can deserialize each - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < families.size(); i++) { - sb.append(families.get(i)); - if (i < families.size() - 1) { - sb.append(SEPARATOR); - } - } - opts.put(INDEX_TO_TABLE_COUNT_KEY, sb.toString()); - Indexer.enableIndexing(desc, ColumnFamilyIndexer.class, opts); - } - - private Map columnTargetMap; - - @Override - public void setup(RegionCoprocessorEnvironment env) { - Configuration conf = env.getConfiguration(); - String[] families = conf.get(INDEX_TO_TABLE_COUNT_KEY).split(SEPARATOR); - - // build up our mapping of column - > index table - columnTargetMap = new HashMap(families.length); - for (int i = 0; i < families.length; i++) { - byte[] fam = Bytes.toBytes(families[i]); - String indexTable = conf.get(INDEX_TO_TABLE_CONF_PREFX + families[i]); - columnTargetMap.put(new ImmutableBytesPtr(fam), indexTable); - } - } - - @Override - public Collection> getIndexUpdate(Put p) { - // if not columns to index, we are done and don't do anything special - if (columnTargetMap == null || columnTargetMap.size() == 0) { - return null; - } - - Collection> updateMap = new ArrayList>(); - Set keys = p.getFamilyMap().keySet(); - for (Entry> entry : p.getFamilyMap().entrySet()) { - String ref = columnTargetMap - .get(new ImmutableBytesPtr(entry.getKey())); - // no reference for that column, skip it - if (ref == null) { - continue; - } - - // get the keys for this family - List kvs = entry.getValue(); - if (kvs == null || kvs.isEmpty()) { - // should never be the case, but just to be careful - continue; - } - - // swap the row key and the column family - Put put = new Put(kvs.get(0).getFamily()); - // got through each of the family's key-values and add it to the put - for (KeyValue kv : entry.getValue()) { - put.add(ColumnFamilyIndexer.INDEX_ROW_COLUMN_FAMILY, - ArrayUtils.addAll(kv.getRow(), kv.getQualifier()), kv.getValue()); - } - - // go through the rest of the families and add them to the put, under the special columnfamily - for (byte[] key : keys) { - if (!Bytes.equals(key, entry.getKey())) { - List otherFamilyKeys = p.getFamilyMap().get(key); - if (otherFamilyKeys == null || otherFamilyKeys.isEmpty()) { - continue; - } - for (KeyValue kv : otherFamilyKeys) { - put.add(ColumnFamilyIndexer.INDEX_REMAINING_COLUMN_FAMILY, - ArrayUtils.addAll(kv.getFamily(), kv.getQualifier()), kv.getValue()); - } - } - } - - // add the mapping - updateMap.add(new Pair(put, Bytes.toBytes(ref))); - } - return updateMap; - } - - @Override - public Collection> getIndexUpdate(Delete d) { - // if no columns to index, we are done and don't do anything special - if (columnTargetMap == null || columnTargetMap.size() == 0) { - return null; - } - - Collection> updateMap = new ArrayList>(); - for (Entry> entry : d.getFamilyMap().entrySet()) { - String ref = columnTargetMap - .get(new ImmutableBytesPtr(entry.getKey())); - // no reference for that column, skip it - if (ref == null) { - continue; - } - List kvs = entry.getValue(); - if (kvs == null || kvs.isEmpty()) { - continue; - } - - // swap the row key and the column family - we only need the row key since we index on the - // column family from the original update - Delete delete = new Delete(kvs.get(0).getFamily()); - // add the mapping - updateMap.add(new Pair(delete, Bytes.toBytes(ref))); - } - return updateMap; - } - - /** - * Create the specified index table with the necessary columns - * @param admin {@link HBaseAdmin} to use when creating the table - * @param indexTable name of the index table. Should be specified in - * {@link setupColumnFamilyIndex} as an index target - */ - public static void createIndexTable(HBaseAdmin admin, String indexTable) throws IOException { - HTableDescriptor index = new HTableDescriptor(indexTable); - index.addFamily(new HColumnDescriptor(INDEX_REMAINING_COLUMN_FAMILY)); - index.addFamily(new HColumnDescriptor(INDEX_ROW_COLUMN_FAMILY)); - - admin.createTable(index); - } - - /** - * Doesn't do any index cleanup. This is just a basic example case. - */ - @Override - public Collection> getIndexUpdateForFilteredRows( - Collection filtered) - throws IOException { - return null; - } -} \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java b/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java index 439d1b31..cf6e290a 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java +++ b/src/main/java/com/salesforce/hbase/index/covered/CoveredColumnsIndexBuilder.java @@ -31,6 +31,8 @@ import java.lang.reflect.Constructor; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -43,13 +45,13 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Mutation; -import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Pair; import com.google.common.collect.Lists; +import com.google.common.primitives.Longs; import com.salesforce.hbase.index.builder.BaseIndexBuilder; import com.salesforce.hbase.index.covered.data.LocalHBaseState; import com.salesforce.hbase.index.covered.data.LocalTable; @@ -63,16 +65,6 @@ * Before any call to prePut/preDelete, the row has already been locked. This ensures that we don't * need to do any extra synchronization in the IndexBuilder. *

- * WARNING: This builder does not correctly support adding the same row twice to a batch - * update. For instance, adding a {@link Put} of the same row twice with different values at - * different timestamps. In this case, the {@link TableState} will return the state of the table - * before the batch; the second Put would not see the earlier Put applied. The reason for - * this is that it is very hard to manage invalidating the local table state in case of failure and - * maintaining a row cache across a batch (see {@link LocalTable} for more information on why). The - * current implementation would manage the above case by possibly not realizing it needed to issue a - * cleanup delete for the index row, leading to an invalid index as one of the updates wouldn't be - * properly covered by a delete. - *

* NOTE: This implementation doesn't cleanup the index when we remove a key-value on compaction or * flush, leading to a bloated index that needs to be cleaned up by a background process. */ @@ -108,14 +100,14 @@ public void setup(RegionCoprocessorEnvironment env) throws IOException { } @Override - public Collection> getIndexUpdate(Put p) throws IOException { + public Collection> getIndexUpdate(Mutation mutation) throws IOException { // build the index updates for each group IndexUpdateManager updateMap = new IndexUpdateManager(); - batchMutationAndAddUpdates(updateMap, p); + batchMutationAndAddUpdates(updateMap, mutation); if (LOG.isDebugEnabled()) { - LOG.debug("Found index updates for Put: " + updateMap); + LOG.debug("Found index updates for Mutation: " + mutation + "\n" + updateMap); } return updateMap.toMap(); @@ -160,16 +152,22 @@ private void batchMutationAndAddUpdates(IndexUpdateManager manager, Mutation m) * {@link KeyValue} with a timestamp == {@link HConstants#LATEST_TIMESTAMP} to the timestamp at * the time the method is called. * @param m {@link Mutation} from which to extract the {@link KeyValue}s - * @return map of timestamp to all the keyvalues with the same timestamp. the implict tree sorting - * in the returned ensures that batches (when iterating through the keys) will iterate the - * kvs in timestamp order + * @return the mutation, broken into batches and sorted in ascending order (smallest first) */ protected Collection createTimestampBatchesFromMutation(Mutation m) { Map batches = new HashMap(); for (List family : m.getFamilyMap().values()) { createTimestampBatchesFromKeyValues(family, batches); } - return batches.values(); + // sort the batches + List sorted = new ArrayList(batches.values()); + Collections.sort(sorted, new Comparator() { + @Override + public int compare(Batch o1, Batch o2) { + return Longs.compare(o1.getTimestamp(), o2.getTimestamp()); + } + }); + return sorted; } /** @@ -177,12 +175,9 @@ protected Collection createTimestampBatchesFromMutation(Mutation m) { * {@link KeyValue} with a timestamp == {@link HConstants#LATEST_TIMESTAMP} to the timestamp at * the time the method is called. * @param kvs {@link KeyValue}s to break into batches - * @param batches - * @return map of timestamp to all the keyvalues with the same timestamp. the implict tree sorting - * in the returned ensures that batches (when iterating through the keys) will iterate the - * kvs in timestamp order + * @param batches to update with the given kvs */ - protected Collection createTimestampBatchesFromKeyValues(Collection kvs, + protected void createTimestampBatchesFromKeyValues(Collection kvs, Map batches) { long now = EnvironmentEdgeManager.currentTimeMillis(); byte[] nowBytes = Bytes.toBytes(now); @@ -202,7 +197,6 @@ protected Collection createTimestampBatchesFromKeyValues(Collection> getIndexUpdate(Delete d) throws IOExce } if (LOG.isDebugEnabled()) { - LOG.debug("Found index updates for Delete: " + updateMap); + LOG.debug("Found index updates for Delete: " + d + "\n" + updateMap); } return updateMap.toMap(); @@ -494,4 +489,10 @@ public Collection> getIndexUpdateForFilteredRows( public void setIndexCodecForTesting(IndexCodec codec) { this.codec = codec; } + + @Override + public boolean isEnabled(Mutation m) { + // ask the codec to see if we should even attempt indexing + return this.codec.isEnabled(m); + } } \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/covered/IndexCodec.java b/src/main/java/com/salesforce/hbase/index/covered/IndexCodec.java index 7a1c84c4..8898c22c 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/IndexCodec.java +++ b/src/main/java/com/salesforce/hbase/index/covered/IndexCodec.java @@ -33,9 +33,15 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import com.salesforce.phoenix.index.BaseIndexCodec; + /** - * Codec for creating index updates from the current state of a table + * Codec for creating index updates from the current state of a table. + *

+ * Generally, you should extend {@link BaseIndexCodec} instead, so help maintain compatibility as + * features need to be added to the codec, as well as potentially not haivng to implement some + * methods. */ public interface IndexCodec { @@ -46,7 +52,6 @@ public interface IndexCodec { */ public void initialize(RegionCoprocessorEnvironment env) throws IOException; - // JY: used for both batch case and covering delete case /** * Get the index cleanup entries. Currently, this must return just single row deletes (where just * the row-key is specified and no columns are returned) mapped to the table name. For instance, @@ -99,4 +104,16 @@ public interface IndexCodec { * basis, as each codec is instantiated per-region. */ public boolean isEnabled(Mutation m); + + /** + * Get the batch identifier of the given mutation. Generally, updates to the table will take place + * in a batch of updates; if we know that the mutation is part of a batch, we can build the state + * much more intelligently. + *

+ * If you have batches that have multiple updates to the same row state, you must specify a + * batch id for each batch. Otherwise, we cannot guarantee index correctness + * @param m mutation that may or may not be part of the batch + * @return null if the mutation is not part of a batch or an id for the batch. + */ + public byte[] getBatchId(Mutation m); } \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/covered/data/IndexMemStore.java b/src/main/java/com/salesforce/hbase/index/covered/data/IndexMemStore.java index 21515144..7a29c3fa 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/data/IndexMemStore.java +++ b/src/main/java/com/salesforce/hbase/index/covered/data/IndexMemStore.java @@ -31,6 +31,8 @@ import java.util.Iterator; import java.util.SortedSet; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KeyComparator; import org.apache.hadoop.hbase.client.Scan; @@ -39,6 +41,7 @@ import org.apache.hadoop.hbase.regionserver.MemStore; import org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner; import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; +import org.apache.hadoop.hbase.util.Bytes; import com.salesforce.hbase.index.covered.KeyValueStore; @@ -74,6 +77,7 @@ */ public class IndexMemStore implements KeyValueStore { + private static final Log LOG = LogFactory.getLog(IndexMemStore.class); private IndexKeyValueSkipListSet kvset; private TimeRangeTracker timeRangeTracker = new TimeRangeTracker(); private Comparator comparator; @@ -107,6 +111,9 @@ public IndexMemStore(Comparator comparator) { @Override public void add(KeyValue kv, boolean overwrite) { + if (LOG.isDebugEnabled()) { + LOG.info("Inserting: " + toString(kv)); + } // if overwriting, we will always update boolean updated = true; if (!overwrite) { @@ -116,13 +123,29 @@ public void add(KeyValue kv, boolean overwrite) { kvset.add(kv); } + // TODO do we even need to update this? I don't think we really even use it // if we updated, we need to do some tracking work if (updated) { // update the max timestamp this.timeRangeTracker.includeTimestamp(kv); + if (LOG.isDebugEnabled()) { + dump(); + } } } + private void dump() { + LOG.debug("Current kv state:\n"); + for (KeyValue kv : this.kvset) { + LOG.debug("KV: " + toString(kv)); + } + LOG.debug("========== END MemStore Dump ==================\n"); + } + + private String toString(KeyValue kv) { + return kv.toString() + "/value=" + Bytes.toString(kv.getValue()); + } + @Override public void rollback(KeyValue kv) { // If the key is in the store, delete it diff --git a/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumn.java b/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumn.java index 257231ff..8c674dbb 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumn.java +++ b/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumn.java @@ -14,15 +14,9 @@ public class CoveredColumn extends ColumnReference { String familyString; private final int hashCode; - private static int calcHashCode(String familyString, byte[] qualifier) { - final int prime = 31; - int result = 1; - result = prime * result + familyString.hashCode(); - if (qualifier != null) { - result = prime * result + Bytes.hashCode(qualifier); - } - return result; - } + public CoveredColumn(byte[] family, byte[] qualifier){ + this(Bytes.toString(family), qualifier); + } public CoveredColumn(String family, byte[] qualifier) { super(Bytes.toBytes(family), qualifier == null ? ColumnReference.ALL_QUALIFIERS : qualifier); @@ -62,22 +56,32 @@ public boolean matchesFamily(String family2) { } @Override -public int hashCode() { + public int hashCode() { return hashCode; -} + } -@Override -public boolean equals(Object obj) { + private static int calcHashCode(String familyString, byte[] qualifier) { + final int prime = 31; + int result = 1; + result = prime * result + familyString.hashCode(); + if (qualifier != null) { + result = prime * result + Bytes.hashCode(qualifier); + } + return result; + } + + @Override + public boolean equals(Object obj) { if (this == obj) return true; if (!super.equals(obj)) return false; if (getClass() != obj.getClass()) return false; - CoveredColumn other = (CoveredColumn)obj; + CoveredColumn other = (CoveredColumn) obj; if (hashCode != other.hashCode) return false; if (!familyString.equals(other.familyString)) return false; return Bytes.equals(qualifier, other.qualifier); -} + } -@Override + @Override public String toString() { String qualString = qualifier == null ? "null" : Bytes.toString(qualifier); return "CoveredColumn:[" + familyString + ":" + qualString + "]"; diff --git a/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexCodec.java b/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexCodec.java index aa51491f..0567b3ab 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexCodec.java +++ b/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexCodec.java @@ -33,15 +33,15 @@ import org.apache.hadoop.hbase.util.Pair; import com.google.common.collect.Lists; -import com.salesforce.hbase.index.covered.IndexCodec; import com.salesforce.hbase.index.covered.IndexUpdate; import com.salesforce.hbase.index.covered.TableState; import com.salesforce.hbase.index.scanner.Scanner; +import com.salesforce.phoenix.index.BaseIndexCodec; /** * */ -public class CoveredColumnIndexCodec implements IndexCodec { +public class CoveredColumnIndexCodec extends BaseIndexCodec { private static final byte[] EMPTY_BYTES = new byte[0]; public static final byte[] INDEX_ROW_COLUMN_FAMILY = Bytes.toBytes("INDEXED_COLUMNS"); diff --git a/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java b/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java index cd82c00f..e781448d 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java +++ b/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java @@ -34,6 +34,7 @@ import java.util.Map; import java.util.Map.Entry; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; @@ -218,18 +219,27 @@ private boolean shouldBeRemoved(Mutation m) { @Override public String toString() { - StringBuffer sb = new StringBuffer("IndexUpdates:\n"); + StringBuffer sb = new StringBuffer("Pending Index Updates:\n"); for (Entry> entry : map.entrySet()) { String tableName = Bytes.toString(entry.getKey().get()); - sb.append(" " + tableName + ":\n"); + sb.append(" Table: '" + tableName + "'\n"); for (Mutation m : entry.getValue()) { sb.append("\t"); if (shouldBeRemoved(m)) { sb.append("[REMOVED]"); } - sb.append("\t" + m.getClass().getSimpleName() + ": " - + ((m instanceof Put) ? m.getTimeStamp() + " " : "") + m); + sb.append(m.getClass().getSimpleName() + ":" + + ((m instanceof Put) ? m.getTimeStamp() + " " : "")); sb.append("\n"); + if (m.getFamilyMap().isEmpty()) { + sb.append("=== EMPTY ==="); + } + for (List kvs : m.getFamilyMap().values()) { + for (KeyValue kv : kvs) { + sb.append("\t\t" + kv.toString() + "/value=" + Bytes.toStringBinary(kv.getValue())); + sb.append("\n"); + } + } } } return sb.toString(); diff --git a/src/main/java/com/salesforce/phoenix/coprocessor/MetaDataEndpointImpl.java b/src/main/java/com/salesforce/phoenix/coprocessor/MetaDataEndpointImpl.java index 9708e144..770e1439 100644 --- a/src/main/java/com/salesforce/phoenix/coprocessor/MetaDataEndpointImpl.java +++ b/src/main/java/com/salesforce/phoenix/coprocessor/MetaDataEndpointImpl.java @@ -476,7 +476,12 @@ public MetaDataMutationResult createTable(List tableMetadata) throws I return new MetaDataMutationResult(MutationCode.NEWER_TABLE_FOUND, EnvironmentEdgeManager.currentTimeMillis(), table); } } - + // TODO: Switch this to HRegion#batchMutate when we want to support indexes on the system + // table. Basically, we get all the locks that we don't already hold for all the + // tableMetadata rows. This ensures we don't have deadlock situations (ensuring primary and + // then index table locks are held, in that order). For now, we just don't support indexing + // on the system table. This is an issue because of the way we manage batch mutation in the + // Indexer. region.mutateRowsWithLocks(tableMetadata, Collections.emptySet()); // Invalidate the cache - the next getTable call will add it diff --git a/src/main/java/com/salesforce/phoenix/index/BaseIndexCodec.java b/src/main/java/com/salesforce/phoenix/index/BaseIndexCodec.java new file mode 100644 index 00000000..05a889ce --- /dev/null +++ b/src/main/java/com/salesforce/phoenix/index/BaseIndexCodec.java @@ -0,0 +1,68 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.phoenix.index; + +import java.io.IOException; + +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; + +import com.salesforce.hbase.index.covered.IndexCodec; + +/** + * + */ +public abstract class BaseIndexCodec implements IndexCodec { + + @Override + public void initialize(RegionCoprocessorEnvironment env) throws IOException { + // noop + } + + /** + * {@inheritDoc} + *

+ * By default, the codec is always enabled. Subclasses should override this method if they want do + * decide to index on a per-mutation basis. + */ + @Override + public boolean isEnabled(Mutation m) { + return true; + } + + /** + * {@inheritDoc} + *

+ * Assumes each mutation is not in a batch. Subclasses that have different batching behavior + * should override this. + */ + @Override + public byte[] getBatchId(Mutation m) { + return null; + } +} \ No newline at end of file diff --git a/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java b/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java index a6dfa72c..291935f2 100644 --- a/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java +++ b/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java @@ -10,7 +10,6 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Set; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; @@ -22,7 +21,6 @@ import org.apache.hadoop.io.WritableUtils; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import com.salesforce.hbase.index.ValueGetter; import com.salesforce.hbase.index.covered.update.ColumnReference; import com.salesforce.hbase.index.util.IndexManagementUtil; @@ -297,7 +295,6 @@ public byte[] buildRowKey(ValueGetter valueGetter, ImmutableBytesWritable rowKey } } - @SuppressWarnings("deprecation") public Put buildUpdateMutation(ValueGetter valueGetter, ImmutableBytesWritable dataRowKeyPtr, long ts) throws IOException { byte[] indexRowKey = this.buildRowKey(valueGetter, dataRowKeyPtr); Put put = new Put(indexRowKey); @@ -309,8 +306,6 @@ public Put buildUpdateMutation(ValueGetter valueGetter, ImmutableBytesWritable d } // Add the empty key value put.add(this.getEmptyKeyValueFamily(), QueryConstants.EMPTY_COLUMN_BYTES, ts, ByteUtil.EMPTY_BYTE_ARRAY); - // TODO: Jesse thinks I should remove this - put.setWriteToWAL(false); return put; } @@ -335,31 +330,38 @@ private boolean indexedColumnsChanged(ValueGetter oldState, ValueGetter newState } return false; } - + @SuppressWarnings("deprecation") public Delete buildDeleteMutation(ValueGetter oldState, ImmutableBytesWritable dataRowKeyPtr, Collection pendingUpdates, long ts) throws IOException { byte[] indexRowKey = this.buildRowKey(oldState, dataRowKeyPtr); // Delete the entire row if any of the indexed columns changed if (pendingUpdates.isEmpty() || indexedColumnsChanged(oldState, IndexManagementUtil.createGetterFromKeyValues(pendingUpdates))) { // Deleting the entire row Delete delete = new Delete(indexRowKey, ts, null); - delete.setWriteToWAL(false); return delete; - } else { // Delete columns for missing key values - Delete delete = new Delete(indexRowKey); - Set pendingRefs = Sets.newHashSetWithExpectedSize(pendingUpdates.size()); - for (KeyValue kv : pendingUpdates) { - pendingRefs.add(new ColumnReference(kv.getFamily(), kv.getQualifier())); + } + // Delete columns for missing key values + Delete delete = new Delete(indexRowKey); + boolean hasColumns = false; + for (KeyValue kv : pendingUpdates) { + if (kv.getType() == KeyValue.Type.Put.getCode()) { + continue; } - for (int i = 0; i < this.getCoverededColumns().size(); i++) { - ColumnReference ref = this.getCoverededColumns().get(i); - if (oldState.getLatestValue(ref) != null && !pendingRefs.contains(ref)) { - byte[] cq = this.indexQualifiers.get(i); - delete.deleteColumns(ref.getFamily(), cq, ts); + hasColumns = true; + byte[] family = kv.getFamily(); + byte[] qual = kv.getQualifier(); + byte[] indexQual = IndexUtil.getIndexColumnName(family, qual); + for (ColumnReference col : this.getCoverededColumns()) { + if (col.matches(kv)) { + delete.deleteColumns(family, indexQual, ts); } - } + } + } + // only return a delete if we need to cleanup some columns + if (hasColumns) { return delete; } - } + return null; + } public byte[] getIndexTableName() { return indexTableName; diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java index b380c60d..20922a9c 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java @@ -16,6 +16,7 @@ package com.salesforce.phoenix.index; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -47,9 +48,11 @@ public void batchStarted(MiniBatchOperationInProgress> m // table rows being indexed into the block cache, as the index maintenance code // does a point scan per row List keys = Lists.newArrayListWithExpectedSize(miniBatchOp.size()); + List maintainers = new ArrayList(); for (int i = 0; i < miniBatchOp.size(); i++) { Mutation m = miniBatchOp.getOperation(i).getFirst(); keys.add(PDataType.VARBINARY.getKeyRange(m.getRow())); + maintainers.addAll(getCodec().getIndexMaintainers(m.getAttributesMap())); } ScanRanges scanRanges = ScanRanges.create(Collections.singletonList(keys), SaltingUtil.VAR_BINARY_SCHEMA); Scan scan = new Scan(); @@ -58,7 +61,6 @@ public void batchStarted(MiniBatchOperationInProgress> m // delete the old index row. We use the columns that we pass through for // the Delete use case, as it includes indexed and covered columns as well // as the empty key value column (which we use to detect a delete of the entire row). - List maintainers = getCodec().getIndexMaintainers(); for (int i = 0; i < maintainers.size(); i++) { IndexMaintainer maintainer = maintainers.get(i); for (int j = 0; j < maintainer.getAllColumns().size(); j++) { @@ -99,4 +101,9 @@ public boolean isEnabled(Mutation m) { // ask the codec to see if we should even attempt indexing return this.codec.isEnabled(m); } + + @Override + public byte[] getBatchId(Mutation m){ + return this.codec.getBatchId(m); + } } \ No newline at end of file diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java index cf7faee0..d11d7482 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java @@ -46,60 +46,43 @@ * {@link #getIndexDeletes(TableState)}) as well as what the new index state should be ( * {@link #getIndexUpserts(TableState)}). */ -public class PhoenixIndexCodec implements IndexCodec { +public class PhoenixIndexCodec extends BaseIndexCodec { public static final String INDEX_MD = "IdxMD"; public static final String INDEX_UUID = "IdxUUID"; - private List indexMaintainers; private final ImmutableBytesWritable ptr = new ImmutableBytesWritable(); private Configuration conf; - private byte[] uuid; @Override public void initialize(RegionCoprocessorEnvironment env) { this.conf = env.getConfiguration(); } - List getIndexMaintainers() { - return indexMaintainers; - } - - /** - * @param m mutation that is being processed - * @return the {@link IndexMaintainer}s that would maintain the index for an update with the - * attributes. - */ - private boolean initIndexMaintainers(Mutation m) { - Map attributes = m.getAttributesMap(); + List getIndexMaintainers(Map attributes){ byte[] uuid = attributes.get(INDEX_UUID); if (uuid == null) { - this.uuid = null; - indexMaintainers = Collections.emptyList(); - return false; - } - if (this.uuid != null && Bytes.compareTo(this.uuid, uuid) == 0) { - return true; + return Collections.emptyList(); } - this.uuid = uuid; - byte[] md = attributes.get(INDEX_MD); + List indexMaintainers; if (md != null) { indexMaintainers = IndexMaintainer.deserialize(md); } else { byte[] tenantIdBytes = attributes.get(PhoenixRuntime.TENANT_ID_ATTRIB); ImmutableBytesWritable tenantId = - tenantIdBytes == null ? null : new ImmutableBytesWritable(tenantIdBytes); + tenantIdBytes == null ? null : new ImmutableBytesWritable(tenantIdBytes); TenantCache cache = GlobalCache.getTenantCache(conf, tenantId); IndexMetaDataCache indexCache = - (IndexMetaDataCache) cache.getServerCache(new ImmutableBytesPtr(uuid)); - this.indexMaintainers = indexCache.getIndexMaintainers(); + (IndexMetaDataCache) cache.getServerCache(new ImmutableBytesPtr(uuid)); + indexMaintainers = indexCache.getIndexMaintainers(); } - return true; + + return indexMaintainers; } @Override public Iterable getIndexUpserts(TableState state) throws IOException { - List indexMaintainers = getIndexMaintainers(); + List indexMaintainers = getIndexMaintainers(state.getUpdateAttributes()); if (indexMaintainers.isEmpty()) { return Collections.emptyList(); } @@ -113,8 +96,7 @@ public Iterable getIndexUpserts(TableState state) throws IOExceptio Scanner scanner = statePair.getFirst(); ValueGetter valueGetter = IndexManagementUtil.createGetterFromScanner(scanner, dataRowKey); ptr.set(dataRowKey); - // TODO: handle Pair because otherwise we'll bloat a sparse covered index - Put put = maintainer.buildUpdateMutation(valueGetter, ptr); + Put put = maintainer.buildUpdateMutation(valueGetter, ptr, state.getCurrentTimestamp()); indexUpdate.setTable(maintainer.getIndexTableName()); indexUpdate.setUpdate(put); //make sure we close the scanner when we are done @@ -126,7 +108,7 @@ public Iterable getIndexUpserts(TableState state) throws IOExceptio @Override public Iterable getIndexDeletes(TableState state) throws IOException { - List indexMaintainers = getIndexMaintainers(); + List indexMaintainers = getIndexMaintainers(state.getUpdateAttributes()); if (indexMaintainers.isEmpty()) { return Collections.emptyList(); } @@ -141,7 +123,9 @@ public Iterable getIndexDeletes(TableState state) throws IOExceptio indexUpdate.setTable(maintainer.getIndexTableName()); ValueGetter valueGetter = IndexManagementUtil.createGetterFromScanner(scanner, dataRowKey); ptr.set(dataRowKey); - Delete delete = maintainer.buildDeleteMutation(valueGetter, ptr, state.getPendingUpdate()); + Delete delete = + maintainer.buildDeleteMutation(valueGetter, ptr, state.getPendingUpdate(), + state.getCurrentTimestamp()); scanner.close(); indexUpdate.setUpdate(delete); indexUpdates.add(indexUpdate); @@ -151,6 +135,12 @@ public Iterable getIndexDeletes(TableState state) throws IOExceptio @Override public boolean isEnabled(Mutation m) { - return initIndexMaintainers(m); + return !getIndexMaintainers(m.getAttributesMap()).isEmpty(); + } + + @Override + public byte[] getBatchId(Mutation m) { + Map attributes = m.getAttributesMap(); + return attributes.get(INDEX_UUID); } } \ No newline at end of file diff --git a/src/main/java/com/salesforce/phoenix/jdbc/PhoenixResultSet.java b/src/main/java/com/salesforce/phoenix/jdbc/PhoenixResultSet.java index e7e12062..3dd97651 100644 --- a/src/main/java/com/salesforce/phoenix/jdbc/PhoenixResultSet.java +++ b/src/main/java/com/salesforce/phoenix/jdbc/PhoenixResultSet.java @@ -1216,4 +1216,9 @@ public T getObject(int columnIndex, Class type) throws SQLException { public T getObject(String columnLabel, Class type) throws SQLException { return (T) getObject(columnLabel); // Just ignore type since we only support built-in types } -} + + @Override + public String toString(){ + return "ResultSet:\n"+ "\tclosed: "+this.isClosed+"\n\tcurrent row: "+currentRow; + } +} \ No newline at end of file diff --git a/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java b/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java index de86a649..c484ffd8 100644 --- a/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java +++ b/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java @@ -489,8 +489,10 @@ private HTableDescriptor generateTableDescriptor(byte[] tableName, HTableDescrip descriptor.addCoprocessor(ServerCachingEndpointImpl.class.getName(), null, 1, null); } // TODO: better encapsulation for this - // Since indexes can't have indexes, don't install our indexing coprocessor for indexes - if (tableType != PTableType.INDEX && !descriptor.hasCoprocessor(Indexer.class.getName())) { + // Since indexes can't have indexes, don't install our indexing coprocessor for indexes. Also, + // don't install on the metadata table until we fix the TODO there. + if (tableType != PTableType.INDEX && !descriptor.hasCoprocessor(Indexer.class.getName()) + && !SchemaUtil.isMetaTable(tableName)) { Map opts = Maps.newHashMapWithExpectedSize(1); opts.put(CoveredColumnsIndexBuilder.CODEC_CLASS_NAME_KEY, PhoenixIndexCodec.class.getName()); Indexer.enableIndexing(descriptor, PhoenixIndexBuilder.class, opts); diff --git a/src/main/java/com/salesforce/phoenix/schema/tuple/ResultTuple.java b/src/main/java/com/salesforce/phoenix/schema/tuple/ResultTuple.java index 3dbe9abf..7afd7148 100644 --- a/src/main/java/com/salesforce/phoenix/schema/tuple/ResultTuple.java +++ b/src/main/java/com/salesforce/phoenix/schema/tuple/ResultTuple.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; import com.salesforce.phoenix.util.ResultUtil; @@ -69,7 +70,24 @@ public KeyValue getValue(byte[] family, byte[] qualifier) { @Override public String toString() { - return result.toString(); + StringBuilder sb = new StringBuilder(); + sb.append("keyvalues="); + if(this.result.isEmpty()) { + sb.append("NONE"); + return sb.toString(); + } + sb.append("{"); + boolean moreThanOne = false; + for(KeyValue kv : this.result.list()) { + if(moreThanOne) { + sb.append(", \n"); + } else { + moreThanOne = true; + } + sb.append(kv.toString()+"/value="+Bytes.toString(kv.getValue())); + } + sb.append("}\n"); + return sb.toString(); } @Override diff --git a/src/test/java/com/salesforce/hbase/index/TestEndtoEndIndexing.java b/src/test/java/com/salesforce/hbase/index/TestEndtoEndIndexing.java deleted file mode 100644 index a5711855..00000000 --- a/src/test/java/com/salesforce/hbase/index/TestEndtoEndIndexing.java +++ /dev/null @@ -1,207 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2013, Salesforce.com, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of Salesforce.com nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ******************************************************************************/ -package com.salesforce.hbase.index; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Mutation; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -import com.salesforce.hbase.index.builder.example.ColumnFamilyIndexer; - -/** - * Test secondary indexing from an end-to-end perspective (client to server to index table) - */ -public class TestEndtoEndIndexing { - - protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); - private static final byte[] FAM = Bytes.toBytes("FAMILY"); - private static final byte[] FAM2 = Bytes.toBytes("FAMILY2"); - private static final String INDEXED_TABLE = "INDEXED_TABLE"; - private static final String INDEX_TABLE = "INDEX_TABLE"; - - @BeforeClass - public static void setupCluster() throws Exception { - Configuration conf = UTIL.getConfiguration(); - IndexTestingUtils.setupConfig(conf); - UTIL.startMiniCluster(); - } - - @AfterClass - public static void teardownCluster() throws Exception { - UTIL.shutdownMiniCluster(); - } - - /** - * Ensure that even if we don't write to the WAL in the Put we at least attempt to index - * the values in the Put - * @throws Exception on failure - */ - @SuppressWarnings("deprecation") - @Test - public void testPutWithoutWALGetsIndexed() throws Exception { - byte[] k = new byte[] { 'a', 'a', 'a' }; - Put put = new Put(k); - put.add(FAM, null, k); - put.add(FAM2, null, k); - put.setWriteToWAL(false); - doPrimaryTablePutWithExpectedIndex(put, 2); - } - - /** - * Test that a simple put into the primary table gets a corresponding put in the index table, in - * non-failure situations. - * @throws Exception on failure - */ - @Test - public void testSimplePrimaryAndIndexTables() throws Exception { - byte[] k = new byte[] { 'a', 'a', 'a' }; - Put put = new Put(k); - put.add(FAM, null, k); - put.add(FAM2, null, k); - doPrimaryTablePutWithExpectedIndex(put, 2); - } - - /** - * Test that we delete change also propagates from the primary table to the index table - * @throws Exception on failure - */ - @Test - public void testPutAndDeleteIsIndexed() throws Exception { - byte[] k = new byte[] { 'a', 'a', 'a' }; - // start with a put, so we know we have some data - Put put = new Put(k); - put.add(FAM, null, k); - put.add(FAM2, null, k); - - // then do a delete of that same row, ending up with no edits in the index table - Delete d = new Delete(k); - // we need to do a full specification here so we in the indexer what to delete on the index - // table - d.deleteColumn(FAM, null); - d.deleteColumn(FAM2, null); - doPrimaryTableUpdatesWithExpectedIndex(Arrays.asList(put, d), 0); - } - - private void doPrimaryTablePutWithExpectedIndex(Put m, int indexSize) throws Exception { - doPrimaryTableUpdatesWithExpectedIndex(Collections.singletonList((Mutation) m), indexSize); - } - - /** - * Create a new primary and index table, write the put to the primary table and then scan the - * index table to ensure that the {@link Put} made it. - * @param put put to write to the primary table - * @param indexSize expected size of the index after the operation - * @throws Exception on failure - */ - private void doPrimaryTableUpdatesWithExpectedIndex(List mutations, int indexSize) - throws Exception { - HTableDescriptor primary = new HTableDescriptor(INDEXED_TABLE); - primary.addFamily(new HColumnDescriptor(FAM)); - primary.addFamily(new HColumnDescriptor(FAM2)); - // setup indexing on one table and one of its columns - Map indexMapping = new HashMap(); - indexMapping.put(FAM, INDEX_TABLE); - ColumnFamilyIndexer.enableIndexing(primary, indexMapping); - - // setup the stats table - HBaseAdmin admin = UTIL.getHBaseAdmin(); - // create the primary table - admin.createTable(primary); - - // create the index table - ColumnFamilyIndexer.createIndexTable(admin, INDEX_TABLE); - - assertTrue("Target index table (" + INDEX_TABLE + ") didn't get created!", - admin.tableExists(INDEX_TABLE)); - - // load some data into our primary table - HTable primaryTable = new HTable(UTIL.getConfiguration(), INDEXED_TABLE); - primaryTable.setAutoFlush(false); - primaryTable.batch(mutations); - primaryTable.flushCommits(); - primaryTable.close(); - - // and now scan the index table - HTable index = new HTable(UTIL.getConfiguration(), INDEX_TABLE); - int count = getKeyValueCount(index); - - // we should have 1 index values - one for each key in the FAM column family - // but none in the FAM2 column family - assertEquals("Got an unexpected amount of index entries!", indexSize, count); - - // then delete the table and make sure we don't have any more stats in our table - admin.disableTable(primary.getName()); - admin.deleteTable(primary.getName()); - admin.disableTable(INDEX_TABLE); - admin.deleteTable(INDEX_TABLE); - } - - /** - * Count the number of keyvalue in the table. Scans all possible versions - * @param table table to scan - * @return number of keyvalues over all rows in the table - * @throws IOException - */ - private int getKeyValueCount(HTable table) throws IOException { - Scan scan = new Scan(); - scan.setMaxVersions(Integer.MAX_VALUE - 1); - - ResultScanner results = table.getScanner(scan); - int count = 0; - for (Result res : results) { - count += res.list().size(); - System.out.println(count + ") " + res); - } - results.close(); - - return count; - } -} diff --git a/src/test/java/com/salesforce/hbase/index/TestFailForUnsupportedHBaseVersions.java b/src/test/java/com/salesforce/hbase/index/TestFailForUnsupportedHBaseVersions.java index 97e6b552..04a61c9c 100644 --- a/src/test/java/com/salesforce/hbase/index/TestFailForUnsupportedHBaseVersions.java +++ b/src/test/java/com/salesforce/hbase/index/TestFailForUnsupportedHBaseVersions.java @@ -31,9 +31,6 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import java.util.HashMap; -import java.util.Map; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -48,7 +45,9 @@ import org.apache.hadoop.hbase.util.VersionInfo; import org.junit.Test; -import com.salesforce.hbase.index.builder.example.ColumnFamilyIndexer; +import com.salesforce.hbase.index.covered.example.ColumnGroup; +import com.salesforce.hbase.index.covered.example.CoveredColumn; +import com.salesforce.hbase.index.covered.example.CoveredColumnIndexSpecifierBuilder; /** * Test that we correctly fail for versions of HBase that don't support current properties @@ -130,13 +129,16 @@ public void testDoesNotStartRegionServerForUnsupportedCompressionAndVersion() th // setup the primary table HTableDescriptor desc = new HTableDescriptor( "testDoesNotStartRegionServerForUnsupportedCompressionAndVersion"); - String family = "f"; - desc.addFamily(new HColumnDescriptor(Bytes.toBytes(family))); + byte[] family = Bytes.toBytes("f"); + desc.addFamily(new HColumnDescriptor(family)); // enable indexing to a non-existant index table - Map familyMap = new HashMap(); - familyMap.put(Bytes.toBytes(family), "INDEX_TABLE"); - ColumnFamilyIndexer.enableIndexing(desc, familyMap); + String indexTableName = "INDEX_TABLE"; + ColumnGroup fam1 = new ColumnGroup(indexTableName); + fam1.add(new CoveredColumn(family, CoveredColumn.ALL_QUALIFIERS)); + CoveredColumnIndexSpecifierBuilder builder = new CoveredColumnIndexSpecifierBuilder(); + builder.addIndexGroup(fam1); + builder.build(desc); // get a reference to the regionserver, so we can ensure it aborts HRegionServer server = util.getMiniHBaseCluster().getRegionServer(0); diff --git a/src/test/java/com/salesforce/hbase/index/TestIndexWriter.java b/src/test/java/com/salesforce/hbase/index/TestIndexWriter.java index 8993e3e1..3e99de57 100644 --- a/src/test/java/com/salesforce/hbase/index/TestIndexWriter.java +++ b/src/test/java/com/salesforce/hbase/index/TestIndexWriter.java @@ -119,7 +119,6 @@ public void correctlyCleanupResources() throws Exception { public void testSynchronouslyCompletesAllWrites() throws Exception { LOG.info("Starting " + testName.getTableNameString()); LOG.info("Current thread is interrupted: " + Thread.interrupted()); - // LOG.info("Current thread is interrupted: " + Thread.interrupted()); Abortable abort = new StubAbortable(); ExecutorService exec = Executors.newFixedThreadPool(1); Map tables = new HashMap(); @@ -166,7 +165,6 @@ public Void answer(InvocationOnMock invocation) throws Throwable { * @throws Exception on failure */ @SuppressWarnings("unchecked") -// @Ignore @Test public void testFailureOnRunningUpdateAbortsPending() throws Exception { Abortable abort = new StubAbortable(); diff --git a/src/test/java/com/salesforce/hbase/index/covered/CoveredIndexCodecForTesting.java b/src/test/java/com/salesforce/hbase/index/covered/CoveredIndexCodecForTesting.java index 3da7fa6d..c1f3680c 100644 --- a/src/test/java/com/salesforce/hbase/index/covered/CoveredIndexCodecForTesting.java +++ b/src/test/java/com/salesforce/hbase/index/covered/CoveredIndexCodecForTesting.java @@ -35,13 +35,13 @@ import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -import com.salesforce.hbase.index.covered.IndexCodec; +import com.salesforce.phoenix.index.BaseIndexCodec; /** * An {@link IndexCodec} for testing that allow you to specify the index updates/deletes, regardless * of the current tables' state. */ -public class CoveredIndexCodecForTesting implements IndexCodec { +public class CoveredIndexCodecForTesting extends BaseIndexCodec { private List deletes = new ArrayList(); private List updates = new ArrayList(); diff --git a/src/test/java/com/salesforce/hbase/index/covered/TestEndToEndCoveredColumnsIndexBuilder.java b/src/test/java/com/salesforce/hbase/index/covered/TestEndToEndCoveredColumnsIndexBuilder.java index 8d41bff3..a3c12489 100644 --- a/src/test/java/com/salesforce/hbase/index/covered/TestEndToEndCoveredColumnsIndexBuilder.java +++ b/src/test/java/com/salesforce/hbase/index/covered/TestEndToEndCoveredColumnsIndexBuilder.java @@ -34,6 +34,7 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -156,7 +157,11 @@ public void verify(TableState state) { int count = 0; KeyValue kv; while ((kv = kvs.next()) != null) { - assertEquals(msg + ": Unexpected kv in table state!", expectedKvs.get(count++), kv); + KeyValue next = expectedKvs.get(count++); + assertEquals( + msg + ": Unexpected kv in table state!\nexpected v1: " + + Bytes.toString(next.getValue()) + "\nactual v1:" + Bytes.toString(kv.getValue()), + next, kv); } assertEquals(msg + ": Didn't find enough kvs in table state!", expectedKvs.size(), count); @@ -240,12 +245,9 @@ public void testExpectedResultsInTableStateForSinglePut() throws Exception { /** * Similar to {@link #testExpectedResultsInTableStateForSinglePut()}, but against batches of puts. - * This case is a little more complicated as we need to use the row cache (based on the response - * from the BatchCache) and keep updating the row cache with each part of the batch. - *

- * This test actually verifies the rather odd behavior documented in the - * {@link PhoenixIndexBuilder} where two updates to the same row in the same batch are completely - * independent in the state of the row they see. + * Previous implementations managed batches by playing current state against each element in the + * batch, rather than combining all the per-row updates into a single mutation for the batch. This + * test ensures that we see the correct expected state. * @throws Exception on failure */ @Test @@ -261,25 +263,29 @@ public void testExpectedResultsInTableStateForBatchPuts() throws Exception { // since we need to iterate the batch. // get all the underlying kvs for the put - final List expectedKvs = new ArrayList(); - final List allKvs = new ArrayList(p1.getFamilyMap().get(family)); + final List allKvs = new ArrayList(2); + allKvs.addAll(p2.getFamilyMap().get(family)); + allKvs.addAll(p1.getFamilyMap().get(family)); // setup the verifier for the data we expect to write - // first call shouldn't have anything in the table + // both puts should be put into a single batch final ColumnReference familyRef = new ColumnReference(TestEndToEndCoveredColumnsIndexBuilder.family, ColumnReference.ALL_QUALIFIERS); VerifyingIndexCodec codec = state.codec; - codec.verifiers.add(new ListMatchingVerifier("cleanup state 1", expectedKvs, familyRef)); - codec.verifiers.add(new ListMatchingVerifier("put state 1", allKvs, familyRef)); - - // second set is completely independent of the first set - List expectedKvs2 = new ArrayList(); - List allKvs2 = new ArrayList(p2.get(family, qual)); - codec.verifiers.add(new ListMatchingVerifier("cleanup state 2", expectedKvs2, familyRef)); - codec.verifiers.add(new ListMatchingVerifier("put state 2", allKvs2, familyRef)); + // no previous state in the table + codec.verifiers.add(new ListMatchingVerifier("cleanup state 1", Collections + . emptyList(), familyRef)); + codec.verifiers.add(new ListMatchingVerifier("put state 1", p1.getFamilyMap().get(family), + familyRef)); + + codec.verifiers.add(new ListMatchingVerifier("cleanup state 2", p1.getFamilyMap().get(family), + familyRef)); + // kvs from both puts should be in the table now + codec.verifiers.add(new ListMatchingVerifier("put state 2", allKvs, familyRef)); // do the actual put (no indexing will actually be done) HTable primary = state.table; + primary.setAutoFlush(false); primary.put(Arrays.asList(p1, p2)); primary.flushCommits(); diff --git a/src/test/java/com/salesforce/hbase/index/covered/example/TestEndToEndCoveredIndexing.java b/src/test/java/com/salesforce/hbase/index/covered/example/TestEndToEndCoveredIndexing.java index 9453b80f..93f0fdf7 100644 --- a/src/test/java/com/salesforce/hbase/index/covered/example/TestEndToEndCoveredIndexing.java +++ b/src/test/java/com/salesforce/hbase/index/covered/example/TestEndToEndCoveredIndexing.java @@ -68,7 +68,7 @@ */ public class TestEndToEndCoveredIndexing { private static final Log LOG = LogFactory.getLog(TestEndToEndCoveredIndexing.class); - private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); private static final String FAM_STRING = "FAMILY"; private static final byte[] FAM = Bytes.toBytes(FAM_STRING); private static final String FAM2_STRING = "FAMILY2"; diff --git a/src/test/java/com/salesforce/hbase/index/TestEndtoEndIndexingWithCompression.java b/src/test/java/com/salesforce/hbase/index/covered/example/TestEndtoEndIndexingWithCompression.java similarity index 86% rename from src/test/java/com/salesforce/hbase/index/TestEndtoEndIndexingWithCompression.java rename to src/test/java/com/salesforce/hbase/index/covered/example/TestEndtoEndIndexingWithCompression.java index 87727aaf..57d53191 100644 --- a/src/test/java/com/salesforce/hbase/index/TestEndtoEndIndexingWithCompression.java +++ b/src/test/java/com/salesforce/hbase/index/covered/example/TestEndtoEndIndexingWithCompression.java @@ -25,7 +25,7 @@ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************/ -package com.salesforce.hbase.index; +package com.salesforce.hbase.index.covered.example; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; @@ -33,17 +33,22 @@ import org.apache.hadoop.hbase.regionserver.wal.WALEditCodec; import org.junit.BeforeClass; +import com.salesforce.hbase.index.IndexTestingUtils; +import com.salesforce.hbase.index.Indexer; /** * Test secondary indexing from an end-to-end perspective (client to server to index table). */ -public class TestEndtoEndIndexingWithCompression extends TestEndtoEndIndexing{ +public class TestEndtoEndIndexingWithCompression extends TestEndToEndCoveredIndexing { @BeforeClass public static void setupCluster() throws Exception { //add our codec and enable WAL compression Configuration conf = UTIL.getConfiguration(); IndexTestingUtils.setupConfig(conf); + // disable version checking, so we can test against whatever version of HBase happens to be + // installed (right now, its generally going to be SNAPSHOT versions). + conf.setBoolean(Indexer.CHECK_VERSION_CONF_KEY, false); conf.set(WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY, IndexedWALEditCodec.class.getName()); conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true); diff --git a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java index 1e567980..c9af92de 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/index/MutableIndexTest.java @@ -133,7 +133,7 @@ public void testCoveredColumnUpdates() throws Exception { stmt.setLong(1,4L); assertEquals(1,stmt.executeUpdate()); conn.commit(); - + rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("chara", rs.getString(1)); @@ -385,7 +385,7 @@ public void testCoveredColumns() throws Exception { assertFalse(rs.next()); } - // @Test Broken, but Jesse is fixing + @Test public void testCompoundIndexKey() throws Exception { String query; ResultSet rs; @@ -393,6 +393,8 @@ public void testCompoundIndexKey() throws Exception { Properties props = new Properties(TEST_PROPERTIES); Connection conn = DriverManager.getConnection(getUrl(), props); conn.setAutoCommit(false); + + // make sure that the tables are empty, but reachable conn.createStatement().execute("CREATE TABLE " + DATA_TABLE_FULL_NAME + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)"); query = "SELECT * FROM " + DATA_TABLE_FULL_NAME; rs = conn.createStatement().executeQuery(query); @@ -403,6 +405,7 @@ public void testCompoundIndexKey() throws Exception { rs = conn.createStatement().executeQuery(query); assertFalse(rs.next()); + // load some data into the table PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?)"); stmt.setString(1,"a"); stmt.setString(2, "x"); @@ -433,16 +436,97 @@ public void testCompoundIndexKey() throws Exception { assertEquals("a",rs.getString(3)); assertFalse(rs.next()); - query = "SELECT * FROM " + DATA_TABLE_FULL_NAME; + query = "SELECT * FROM " + DATA_TABLE_FULL_NAME; rs = conn.createStatement().executeQuery("EXPLAIN " + query); assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + INDEX_TABLE_FULL_NAME, QueryUtil.getExplainPlan(rs)); - + //make sure the data table looks like what we expect rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("a",rs.getString(1)); - assertNull(rs.getString(2)); - assertEquals("y",rs.getString(3)); + assertEquals("y",rs.getString(2)); + assertNull(rs.getString(3)); assertFalse(rs.next()); } + + /** + * There was a case where if there were multiple updates to a single row in the same batch, the + * index wouldn't be updated correctly as each element of the batch was evaluated with the state + * previous to the batch, rather than with the rest of the batch. This meant you could do a put + * and a delete on a row in the same batch and the index result would contain the current + put + * and current + delete, but not current + put + delete. + * @throws Exception on failure + */ + @Test + public void testMultipleUpdatesToSingleRow() throws Exception { + String query; + ResultSet rs; + + Properties props = new Properties(TEST_PROPERTIES); + Connection conn = DriverManager.getConnection(getUrl(), props); + conn.setAutoCommit(false); + + // make sure that the tables are empty, but reachable + conn.createStatement().execute( + "CREATE TABLE " + DATA_TABLE_FULL_NAME + + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)"); + query = "SELECT * FROM " + DATA_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertFalse(rs.next()); + + conn.createStatement().execute( + "CREATE INDEX " + INDEX_TABLE_NAME + " ON " + DATA_TABLE_FULL_NAME + " (v1, v2)"); + query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertFalse(rs.next()); -} + // load some data into the table + PreparedStatement stmt = + conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?)"); + stmt.setString(1, "a"); + stmt.setString(2, "x"); + stmt.setString(3, "1"); + stmt.execute(); + conn.commit(); + + // make sure the index is working as expected + query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("x", rs.getString(1)); + assertEquals("1", rs.getString(2)); + assertEquals("a", rs.getString(3)); + assertFalse(rs.next()); + + // do multiple updates to the same row, in the same batch + stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + "(k, v1) VALUES(?,?)"); + stmt.setString(1, "a"); + stmt.setString(2, "y"); + stmt.execute(); + stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + "(k,v2) VALUES(?,?)"); + stmt.setString(1, "a"); + stmt.setString(2, null); + stmt.execute(); + conn.commit(); + + query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("y", rs.getString(1)); + assertNull(rs.getString(2)); + assertEquals("a", rs.getString(3)); + assertFalse(rs.next()); + + query = "SELECT * FROM " + DATA_TABLE_FULL_NAME; + rs = conn.createStatement().executeQuery("EXPLAIN " + query); + assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + INDEX_TABLE_FULL_NAME, + QueryUtil.getExplainPlan(rs)); + + // check that the data table matches as expected + rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("a", rs.getString(1)); + assertEquals("y", rs.getString(2)); + assertNull(rs.getString(3)); + assertFalse(rs.next()); + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWrites.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWrites.java index e445ab26..4935a2e1 100644 --- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWrites.java +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWrites.java @@ -4,9 +4,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -34,11 +32,16 @@ import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; import org.mockito.Mockito; import com.salesforce.hbase.index.IndexTestingUtils; -import com.salesforce.hbase.index.builder.example.ColumnFamilyIndexer; +import com.salesforce.hbase.index.TableName; +import com.salesforce.hbase.index.covered.example.ColumnGroup; +import com.salesforce.hbase.index.covered.example.CoveredColumn; +import com.salesforce.hbase.index.covered.example.CoveredColumnIndexSpecifierBuilder; +import com.salesforce.hbase.index.covered.example.CoveredColumnIndexer; /** * most of the underlying work (creating/splitting the WAL, etc) is from @@ -49,7 +52,10 @@ public class TestWALReplayWithIndexWrites { public static final Log LOG = LogFactory.getLog(TestWALReplay.class); static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); - private static final String INDEX_TABLE_NAME = "IndexTable"; + @Rule + public TableName table = new TableName(); + private String INDEX_TABLE_NAME = table.getTableNameString() + "_INDEX"; + private Path hbaseRootDir = null; private Path oldLogDir; private Path logDir; @@ -131,10 +137,13 @@ public void testReplayEditsWrittenViaHRegion() throws Exception { final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr); //setup basic indexing for the table - Map familyMap = new HashMap(); - byte[] indexedFamily = new byte[] {'a'}; - familyMap.put(indexedFamily, INDEX_TABLE_NAME); - ColumnFamilyIndexer.enableIndexing(htd, familyMap); + // enable indexing to a non-existant index table + byte[] family = new byte[] { 'a' }; + ColumnGroup fam1 = new ColumnGroup(INDEX_TABLE_NAME); + fam1.add(new CoveredColumn(family, CoveredColumn.ALL_QUALIFIERS)); + CoveredColumnIndexSpecifierBuilder builder = new CoveredColumnIndexSpecifierBuilder(); + builder.addIndexGroup(fam1); + builder.build(htd); // create the region + its WAL HRegion region0 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd); @@ -154,8 +163,8 @@ public void testReplayEditsWrittenViaHRegion() throws Exception { //make an attempted write to the primary that should also be indexed byte[] rowkey = Bytes.toBytes("indexed_row_key"); Put p = new Put(rowkey); - p.add(indexedFamily, Bytes.toBytes("qual"), Bytes.toBytes("value")); - region.put(p); + p.add(family, Bytes.toBytes("qual"), Bytes.toBytes("value")); + region.put(new Put[] { p }); // we should then see the server go down Mockito.verify(mockRS, Mockito.times(1)).abort(Mockito.anyString(), @@ -164,7 +173,7 @@ public void testReplayEditsWrittenViaHRegion() throws Exception { wal.close(); // then create the index table so we are successful on WAL replay - ColumnFamilyIndexer.createIndexTable(UTIL.getHBaseAdmin(), INDEX_TABLE_NAME); + CoveredColumnIndexer.createIndexTable(UTIL.getHBaseAdmin(), INDEX_TABLE_NAME); // run the WAL split and setup the region runWALSplit(this.conf); From 7520b78848005cc06b060122ba8cea900e03603d Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Tue, 24 Sep 2013 16:46:30 -0700 Subject: [PATCH 068/102] Making index writing pluggable. There are two main pluggability points: (1) writing the index and (2) handling failure. This provides the option to mix and match choices. For instance, it would be pretty easy to write an AsyncHBase writer and use the standad KillServerOnFailurePolicy to ensure that the server fails if all index writes don't succeed. By default, we retain the exsiting behavior, which uses a thread-pool to write to a cache of HTables and kills the server if any write fails. --- .../index/CannotReachIndexException.java | 11 + .../com/salesforce/hbase/index/Indexer.java | 4 +- .../hbase/index/write/IndexCommitter.java | 47 ++++ .../hbase/index/write/IndexFailurePolicy.java | 51 +++++ .../hbase/index/write/IndexWriter.java | 207 ++++++++++++++++++ .../write/KillServerOnFailurePolicy.java | 88 ++++++++ .../ParallelWriterIndexCommitter.java} | 190 ++++------------ .../hbase/index/write/FakeTableFactory.java | 33 +++ .../index/{ => write}/TestIndexWriter.java | 74 ++----- .../index/write/TestParalleIndexWriter.java | 129 +++++++++++ 10 files changed, 633 insertions(+), 201 deletions(-) create mode 100644 src/main/java/com/salesforce/hbase/index/write/IndexCommitter.java create mode 100644 src/main/java/com/salesforce/hbase/index/write/IndexFailurePolicy.java create mode 100644 src/main/java/com/salesforce/hbase/index/write/IndexWriter.java create mode 100644 src/main/java/com/salesforce/hbase/index/write/KillServerOnFailurePolicy.java rename src/main/java/com/salesforce/hbase/index/{IndexWriter.java => write/ParallelWriterIndexCommitter.java} (65%) create mode 100644 src/test/java/com/salesforce/hbase/index/write/FakeTableFactory.java rename src/test/java/com/salesforce/hbase/index/{ => write}/TestIndexWriter.java (85%) create mode 100644 src/test/java/com/salesforce/hbase/index/write/TestParalleIndexWriter.java diff --git a/src/main/java/com/salesforce/hbase/index/CannotReachIndexException.java b/src/main/java/com/salesforce/hbase/index/CannotReachIndexException.java index 5097cfe2..895e4a0a 100644 --- a/src/main/java/com/salesforce/hbase/index/CannotReachIndexException.java +++ b/src/main/java/com/salesforce/hbase/index/CannotReachIndexException.java @@ -37,6 +37,8 @@ @SuppressWarnings("serial") public class CannotReachIndexException extends Exception { + private String table; + /** * Cannot reach the index, but not sure of the table or the mutations that caused the failure * @param msg more description of what happened @@ -55,5 +57,14 @@ public CannotReachIndexException(String msg, Throwable cause) { public CannotReachIndexException(String targetTableName, List mutations, Exception cause) { super("Failed to make index update:\n\t table: " + targetTableName + "\n\t edits: " + mutations + "\n\tcause: " + cause == null ? "UNKNOWN" : cause.getMessage(), cause); + this.table = targetTableName; + } + + /** + * @return The table to which we failed to write the index updates. If unknown, returns + * null + */ + public String getTableName() { + return this.table; } } \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/Indexer.java b/src/main/java/com/salesforce/hbase/index/Indexer.java index 1d102392..4b000018 100644 --- a/src/main/java/com/salesforce/hbase/index/Indexer.java +++ b/src/main/java/com/salesforce/hbase/index/Indexer.java @@ -68,6 +68,7 @@ import com.salesforce.hbase.index.builder.IndexBuilder; import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.hbase.index.wal.IndexedKeyValue; +import com.salesforce.hbase.index.write.IndexWriter; /** * Do all the work of managing index updates from a single coprocessor. All Puts/Delets are passed @@ -149,8 +150,7 @@ public void start(CoprocessorEnvironment e) throws IOException { log.registerWALActionsListener(new IndexLogRollSynchronizer(INDEX_READ_WRITE_LOCK.writeLock())); // and setup the actual index writer - this.writer = new IndexWriter("Region: " + env.getRegion().getRegionNameAsString(), - env.getRegionServerServices(), env, conf); + this.writer = new IndexWriter(env); } @Override diff --git a/src/main/java/com/salesforce/hbase/index/write/IndexCommitter.java b/src/main/java/com/salesforce/hbase/index/write/IndexCommitter.java new file mode 100644 index 00000000..a763f329 --- /dev/null +++ b/src/main/java/com/salesforce/hbase/index/write/IndexCommitter.java @@ -0,0 +1,47 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index.write; + +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; + +import com.google.common.collect.Multimap; +import com.salesforce.hbase.index.CannotReachIndexException; +import com.salesforce.hbase.index.table.HTableInterfaceReference; + +/** + * Write the index updates to the index tables + */ +public interface IndexCommitter extends Stoppable { + + void setup(IndexWriter parent, RegionCoprocessorEnvironment env); + + public void write(Multimap toWrite) + throws CannotReachIndexException; +} \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/write/IndexFailurePolicy.java b/src/main/java/com/salesforce/hbase/index/write/IndexFailurePolicy.java new file mode 100644 index 00000000..b8de06c9 --- /dev/null +++ b/src/main/java/com/salesforce/hbase/index/write/IndexFailurePolicy.java @@ -0,0 +1,51 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index.write; + +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; + +import com.google.common.collect.Multimap; +import com.salesforce.hbase.index.table.HTableInterfaceReference; + +/** + * Handle failures to write to the index tables. + */ +public interface IndexFailurePolicy extends Stoppable { + + public void setup(Stoppable parent, RegionCoprocessorEnvironment env); + + /** + * Handle the failure of the attempted index updates + * @param attempted map of target table -> mutations to apply + * @param cause reason why there was a failure + */ + public void + handleFailure(Multimap attempted, Exception cause); +} \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/write/IndexWriter.java b/src/main/java/com/salesforce/hbase/index/write/IndexWriter.java new file mode 100644 index 00000000..e1b1d0a6 --- /dev/null +++ b/src/main/java/com/salesforce/hbase/index/write/IndexWriter.java @@ -0,0 +1,207 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index.write; + +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.util.Pair; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; +import com.salesforce.hbase.index.CannotReachIndexException; +import com.salesforce.hbase.index.table.HTableInterfaceReference; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; + +/** + * Do the actual work of writing to the index tables. Ensures that if we do fail to write to the + * index table that we cleanly kill the region/server to ensure that the region's WAL gets replayed. + *

+ * We attempt to do the index updates in parallel using a backing threadpool. All threads are daemon + * threads, so it will not block the region from shutting down. + */ +public class IndexWriter implements Stoppable { + + private static final Log LOG = LogFactory.getLog(IndexWriter.class); + private static final String INDEX_COMMITTER_CONF_KEY = null; + private AtomicBoolean stopped = new AtomicBoolean(false); + private IndexCommitter writer; + private IndexFailurePolicy failurePolicy; + + /** + * @throws IOException if the {@link IndexWriter} or {@link IndexFailurePolicy} cannot be + * instantiated + */ + public IndexWriter(RegionCoprocessorEnvironment env) throws IOException { + this(getCommitter(env), getFailurePolicy(env)); + this.writer.setup(this, env); + this.failurePolicy.setup(this, env); + } + + private static IndexCommitter getCommitter(RegionCoprocessorEnvironment env) throws IOException { + Configuration conf = env.getConfiguration(); + try { + IndexCommitter committer = + conf.getClass(INDEX_COMMITTER_CONF_KEY, ParallelWriterIndexCommitter.class, + IndexCommitter.class).newInstance(); + return committer; + } catch (InstantiationException e) { + throw new IOException(e); + } catch (IllegalAccessException e) { + throw new IOException(e); + } + } + + private static IndexFailurePolicy getFailurePolicy(RegionCoprocessorEnvironment env) + throws IOException { + Configuration conf = env.getConfiguration(); + try { + IndexFailurePolicy committer = + conf.getClass(INDEX_COMMITTER_CONF_KEY, KillServerOnFailurePolicy.class, + IndexFailurePolicy.class).newInstance(); + return committer; + } catch (InstantiationException e) { + throw new IOException(e); + } catch (IllegalAccessException e) { + throw new IOException(e); + } + } + + /** + * Exposed for TESTING! Does no setup of the {@link IndexFailurePolicy} or {@link IndexCommitter}. + */ + IndexWriter(IndexCommitter committer, IndexFailurePolicy policy) { + this.writer = committer; + this.failurePolicy = policy; + } + /** + * Write the mutations to their respective table. + *

+ * This method is blocking and could potentially cause the writer to block for a long time as we + * write the index updates. We only return when either: + *

    + *
  1. All index writes have returned, OR
    2. + *
  2. Any single index write has failed
    3. + *
+ * We attempt to quickly determine if any write has failed and not write to the remaining indexes + * to ensure a timely recovery of the failed index writes. + *

+ * If any of the index updates fails, we pass along the failure to the installed + * {@link IndexFailurePolicy}, which then decides how to handle the failure. By default, we use a + * {@link KillServerOnFailurePolicy}, which ensures that the server crashes when an index write + * fails, ensuring that we get WAL replay of the index edits. + * @param indexUpdates Updates to write + */ + public void writeAndKillYourselfOnFailure(Collection> indexUpdates) { + // convert the strings to htableinterfaces to which we can talk and group by TABLE + Multimap toWrite = resolveTableReferences(indexUpdates); + try { + this.writer.write(toWrite); + LOG.info("Done writing all index updates!"); + } catch (Exception e) { + this.failurePolicy.handleFailure(toWrite, e); + } + } + + /** + * Write the mutations to their respective table. + *

+ * This method is blocking and could potentially cause the writer to block for a long time as we + * write the index updates. We only return when either: + *

    + *
  1. All index writes have returned, OR
    2. + *
  2. Any single index write has failed
    3. + *
+ * We attempt to quickly determine if any write has failed and not write to the remaining indexes + * to ensure a timely recovery of the failed index writes. + * @param toWrite Updates to write + * @throws CannotReachIndexException if we cannot successfully write a single index entry. We stop + * immediately on the first failed index write, rather than attempting all writes. + */ + public void write(Collection> toWrite) throws CannotReachIndexException { + this.writer.write(resolveTableReferences(toWrite)); + } + + + public void write(Multimap toWrite) + throws CannotReachIndexException { + + } + + + /** + * Convert the passed index updates to {@link HTableInterfaceReference}s. + * @param indexUpdates from the index builder + * @return pairs that can then be written by an {@link IndexWriter}. + */ + public static Multimap resolveTableReferences( + Collection> indexUpdates) { + Multimap updates = ArrayListMultimap + . create(); + // simple map to make lookups easy while we build the map of tables to create + Map tables = + new HashMap(updates.size()); + for (Pair entry : indexUpdates) { + byte[] tableName = entry.getSecond(); + ImmutableBytesPtr ptr = new ImmutableBytesPtr(tableName); + HTableInterfaceReference table = tables.get(ptr); + if (table == null) { + table = new HTableInterfaceReference(ptr); + tables.put(ptr, table); + } + updates.put(table, entry.getFirst()); + } + + return updates; + } + + @Override + public void stop(String why) { + if (!this.stopped.compareAndSet(false, true)) { + // already stopped + return; + } + LOG.debug("Stopping because " + why); + this.writer.stop(why); + this.failurePolicy.stop(why); + } + + @Override + public boolean isStopped() { + return this.stopped.get(); + } +} \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/write/KillServerOnFailurePolicy.java b/src/main/java/com/salesforce/hbase/index/write/KillServerOnFailurePolicy.java new file mode 100644 index 00000000..73865270 --- /dev/null +++ b/src/main/java/com/salesforce/hbase/index/write/KillServerOnFailurePolicy.java @@ -0,0 +1,88 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index.write; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; + +import com.google.common.collect.Multimap; +import com.salesforce.hbase.index.table.HTableInterfaceReference; + +/** + * Naive failure policy - kills the server on which it resides + */ +public class KillServerOnFailurePolicy implements IndexFailurePolicy { + + private static final Log LOG = LogFactory.getLog(KillServerOnFailurePolicy.class); + private Abortable abortable; + private Stoppable stoppable; + + @Override + public void setup(Stoppable parent, RegionCoprocessorEnvironment env) { + setup(parent, env.getRegionServerServices()); + } + + public void setup(Stoppable parent, Abortable abort) { + this.stoppable = parent; + this.abortable = abort; + } + + @Override + public void stop(String why) { + // noop + } + + @Override + public boolean isStopped() { + return this.stoppable.isStopped(); + } + + @Override + public void + handleFailure(Multimap attempted, Exception cause) { + // cleanup resources + this.stop("Killing ourselves because of an error:" + cause); + // notify the regionserver of the failure + String msg = + "Could not update the index table, killing server region because couldn't write to an index table"; + LOG.error(msg, cause); + try { + this.abortable.abort(msg, cause); + } catch (Exception e) { + LOG.fatal("Couldn't abort this server to preserve index writes, " + + "attempting to hard kill the server"); + System.exit(1); + } + + } + +} diff --git a/src/main/java/com/salesforce/hbase/index/IndexWriter.java b/src/main/java/com/salesforce/hbase/index/write/ParallelWriterIndexCommitter.java similarity index 65% rename from src/main/java/com/salesforce/hbase/index/IndexWriter.java rename to src/main/java/com/salesforce/hbase/index/write/ParallelWriterIndexCommitter.java index 22aefb0f..72482ea2 100644 --- a/src/main/java/com/salesforce/hbase/index/IndexWriter.java +++ b/src/main/java/com/salesforce/hbase/index/write/ParallelWriterIndexCommitter.java @@ -25,13 +25,11 @@ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************/ -package com.salesforce.hbase.index; +package com.salesforce.hbase.index.write; import java.io.IOException; import java.util.Collection; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.Callable; @@ -44,7 +42,6 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -52,56 +49,64 @@ import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.Stoppable; -import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -import org.apache.hadoop.hbase.regionserver.wal.IndexedWALEdit; -import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Threads; -import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import com.salesforce.hbase.index.CannotReachIndexException; +import com.salesforce.hbase.index.CapturingAbortable; import com.salesforce.hbase.index.table.CachingHTableFactory; import com.salesforce.hbase.index.table.CoprocessorHTableFactory; import com.salesforce.hbase.index.table.HTableFactory; import com.salesforce.hbase.index.table.HTableInterfaceReference; -import com.salesforce.hbase.index.util.ImmutableBytesPtr; /** - * Do the actual work of writing to the index tables. Ensures that if we do fail to write to the - * index table that we cleanly kill the region/server to ensure that the region's WAL gets replayed. - *

- * We attempt to do the index updates in parallel using a backing threadpool. All threads are daemon - * threads, so it will not block the region from shutting down. + * */ -public class IndexWriter implements Stoppable { +public class ParallelWriterIndexCommitter implements IndexCommitter { + + private static final Log LOG = LogFactory.getLog(ParallelWriterIndexCommitter.class); public static String NUM_CONCURRENT_INDEX_WRITER_THREADS_CONF_KEY = "index.writer.threads.max"; - private static final Log LOG = LogFactory.getLog(IndexWriter.class); private static final int DEFAULT_CONCURRENT_INDEX_WRITER_THREADS = 10; private static final String INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY = "index.writer.threads.keepalivetime"; - private final String sourceInfo; - private final CapturingAbortable abortable; - private final HTableFactory factory; private ListeningExecutorService writerPool; - private AtomicBoolean stopped = new AtomicBoolean(false); + private HTableFactory factory; + private CapturingAbortable abortable; + private Stoppable stopped; + + @Override + public void setup(IndexWriter parent, RegionCoprocessorEnvironment env) { + Configuration conf = env.getConfiguration(); + setup(getDefaultDelegateHTableFactory(env), getDefaultExecutor(conf), + env.getRegionServerServices(), parent); + } /** - * @param sourceInfo log info string about where we are writing from - * @param abortable to notify in the case of failure - * @param env Factory to use when resolving the {@link HTableInterfaceReference}. If null - * , its assumed that the {@link HTableInterfaceReference} already has its factory set - * (e.g. by {@link HTableInterfaceReference#setFactory(HTableFactory)} - if its not - * already set, a {@link NullPointerException} is thrown. + * Setup this. + *

+ * Exposed for TESTING */ - public IndexWriter(String sourceInfo, Abortable abortable, RegionCoprocessorEnvironment env, - Configuration conf) { - this(sourceInfo, abortable, env, getDefaultExecutor(conf)); + void setup(HTableFactory factory, ExecutorService pool, Abortable abortable, Stoppable stop) { + this.writerPool = MoreExecutors.listeningDecorator(pool); + this.factory = new CachingHTableFactory(factory); + this.abortable = new CapturingAbortable(abortable); + this.stopped = stop; + } + + private static HTableFactory getDefaultDelegateHTableFactory(CoprocessorEnvironment env) { + // create a simple delegate factory, setup the way we need + Configuration conf = env.getConfiguration(); + // only have one thread per table - all the writes are already batched per table. + + conf.setInt("hbase.htable.threads.max", 1); + return new CoprocessorHTableFactory(env); } /** @@ -133,72 +138,9 @@ private static ThreadPoolExecutor getDefaultExecutor(Configuration conf) { return pool; } - public IndexWriter(String sourceInfo, Abortable abortable, CoprocessorEnvironment env, - ExecutorService pool) { - this(sourceInfo, abortable, pool, getDefaultDelegateHTableFactory(env)); - } - - private static HTableFactory getDefaultDelegateHTableFactory(CoprocessorEnvironment env) { - // create a simple delegate factory, setup the way we need - Configuration conf = env.getConfiguration(); - // only have one thread per table - all the writes are already batched per table. - - conf.setInt("hbase.htable.threads.max", 1); - return new CoprocessorHTableFactory(env); - } - - /** - * Internal constructor - Exposed for testing! - * @param sourceInfo - * @param abortable - * @param pool - * @param delegate - */ - IndexWriter(String sourceInfo, Abortable abortable, ExecutorService pool, HTableFactory delegate) { - this.sourceInfo = sourceInfo; - this.abortable = new CapturingAbortable(abortable); - this.writerPool = MoreExecutors.listeningDecorator(pool); - this.factory = new CachingHTableFactory(delegate); - } - - /** - * Just write the index update portions of of the edit, if it is an {@link IndexedWALEdit}. If it - * is not passed an {@link IndexedWALEdit}, any further actions are ignored. - *

- * Internally, uses {@link #write(Collection)} to make the write and if is receives a - * {@link CannotReachIndexException}, it attempts to move ( - * {@link HBaseAdmin#unassign(byte[], boolean)}) the region and then failing that calls - * {@link System#exit(int)} to kill the server. - */ - public void writeAndKillYourselfOnFailure(Collection> indexUpdates) { - try { - write(indexUpdates); - } catch (Exception e) { - killYourself(e); - } - } - - /** - * Write the mutations to their respective table. - *

- * This method is blocking and could potentially cause the writer to block for a long time as we - * write the index updates. We only return when either: - *

    - *
  1. All index writes have returned, OR
    2. - *
  2. Any single index write has failed
    3. - *
- * We attempt to quickly determine if any write has failed and not write to the remaining indexes - * to ensure a timely recovery of the failed index writes. - * @param indexUpdates Updates to write - * @throws CannotReachIndexException if we cannot successfully write a single index entry. We stop - * immediately on the first failed index write, rather than attempting all writes. - */ - public void write(Collection> indexUpdates) + @Override + public void write(Multimap toWrite) throws CannotReachIndexException { - // convert the strings to htableinterfaces to which we can talk and group by TABLE - Multimap toWrite = resolveTableReferences(factory, - indexUpdates); - /* * This bit here is a little odd, so let's explain what's going on. Basically, we want to do the * writes in parallel to each index table, so each table gets its own task and is submitted to @@ -218,7 +160,7 @@ public void write(Collection> indexUpdates) final List mutations = (List) entry.getValue(); final HTableInterfaceReference tableReference = entry.getKey(); // early exit - no need to submit new tasks if we are shutting down - if (this.stopped.get() || this.abortable.isAborted()) { + if (this.stopped.isStopped() || this.abortable.isAborted()) { break; } @@ -264,7 +206,8 @@ public Void call() throws Exception { } private void throwFailureIfDone() throws CannotReachIndexException { - if (stopped.get() || abortable.isAborted() || Thread.currentThread().isInterrupted()) { + if (stopped.isStopped() || abortable.isAborted() + || Thread.currentThread().isInterrupted()) { throw new CannotReachIndexException( "Pool closed, not attempting to write to the index!", null); } @@ -321,68 +264,17 @@ private void throwFailureIfDone() throws CannotReachIndexException { e); } - LOG.info("Done writing all index updates!"); - } - - /** - * @param logEdit edit for which we need to kill ourselves - * @param info region from which we are attempting to write the log - */ - private void killYourself(Throwable cause) { - // cleanup resources - this.stop("Killing ourselves because of an error:" + cause); - // notify the regionserver of the failure - String msg = "Could not update the index table, killing server region from: " + this.sourceInfo; - LOG.error(msg); - try { - this.abortable.abort(msg, cause); - } catch (Exception e) { - LOG.fatal("Couldn't abort this server to preserve index writes, " - + "attempting to hard kill the server from" + this.sourceInfo); - System.exit(1); - } - } - - /** - * Convert the passed index updates to {@link HTableInterfaceReference}s. - * @param factory factory to use when resolving the table references. - * @param indexUpdates from the index builder - * @return pairs that can then be written by an {@link IndexWriter}. - */ - public static Multimap resolveTableReferences( - HTableFactory factory, Collection> indexUpdates) { - Multimap updates = ArrayListMultimap - . create(); - // simple map to make lookups easy while we build the map of tables to create - Map tables = - new HashMap(updates.size()); - for (Pair entry : indexUpdates) { - byte[] tableName = entry.getSecond(); - ImmutableBytesPtr ptr = new ImmutableBytesPtr(tableName); - HTableInterfaceReference table = tables.get(ptr); - if (table == null) { - table = new HTableInterfaceReference(ptr); - tables.put(ptr, table); - } - updates.put(table, entry.getFirst()); - } - - return updates; } @Override public void stop(String why) { - if (!this.stopped.compareAndSet(false, true)) { - // already stopped - return; - } - LOG.debug("Stopping because " + why); + LOG.info("Shutting down " + this.getClass().getSimpleName()); this.writerPool.shutdownNow(); this.factory.shutdown(); } @Override public boolean isStopped() { - return this.stopped.get(); + return this.stopped.isStopped(); } } \ No newline at end of file diff --git a/src/test/java/com/salesforce/hbase/index/write/FakeTableFactory.java b/src/test/java/com/salesforce/hbase/index/write/FakeTableFactory.java new file mode 100644 index 00000000..22f90a91 --- /dev/null +++ b/src/test/java/com/salesforce/hbase/index/write/FakeTableFactory.java @@ -0,0 +1,33 @@ +package com.salesforce.hbase.index.write; + +import java.io.IOException; +import java.util.Map; + +import org.apache.hadoop.hbase.client.HTableInterface; + +import com.salesforce.hbase.index.table.HTableFactory; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; + +/** + * Simple table factory that just looks up the tables based on name. Useful for mocking up + * {@link HTableInterface}s without having to mock up the factory too. + */ +class FakeTableFactory implements HTableFactory { + + boolean shutdown = false; + private Map tables; + + public FakeTableFactory(Map tables) { + this.tables = tables; + } + + @Override + public HTableInterface getTable(ImmutableBytesPtr tablename) throws IOException { + return this.tables.get(tablename); + } + + @Override + public void shutdown() { + shutdown = true; + } +} \ No newline at end of file diff --git a/src/test/java/com/salesforce/hbase/index/TestIndexWriter.java b/src/test/java/com/salesforce/hbase/index/write/TestIndexWriter.java similarity index 85% rename from src/test/java/com/salesforce/hbase/index/TestIndexWriter.java rename to src/test/java/com/salesforce/hbase/index/write/TestIndexWriter.java index 3e99de57..94d5b284 100644 --- a/src/test/java/com/salesforce/hbase/index/TestIndexWriter.java +++ b/src/test/java/com/salesforce/hbase/index/write/TestIndexWriter.java @@ -25,7 +25,7 @@ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************/ -package com.salesforce.hbase.index; +package com.salesforce.hbase.index.write; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -35,7 +35,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -46,6 +45,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; @@ -57,57 +57,17 @@ import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import com.salesforce.hbase.index.table.HTableFactory; +import com.salesforce.hbase.index.CannotReachIndexException; +import com.salesforce.hbase.index.StubAbortable; +import com.salesforce.hbase.index.TableName; import com.salesforce.hbase.index.util.ImmutableBytesPtr; public class TestIndexWriter { private static final Log LOG = LogFactory.getLog(TestIndexWriter.class); @Rule public TableName testName = new TableName(); - - /** - * Simple table factory that just looks up the tables based on name. Useful for mocking up - * {@link HTableInterface}s without having to mock up the factory too. - */ - private class FakeTableFactory implements HTableFactory { - - boolean shutdown = false; - private Map tables; - - public FakeTableFactory(Map tables) { - this.tables = tables; - } - - @Override - public HTableInterface getTable(ImmutableBytesPtr tablename) throws IOException { - return this.tables.get(tablename); - } - - @Override - public void shutdown() { - shutdown = true; - } - } - private final byte[] row = Bytes.toBytes("row"); - /** - * Test that we correctly shutdown/cleanup all the resources the writer creates - * @throws Exception on failure - */ - @Test - public void correctlyCleanupResources() throws Exception { - Abortable abort = new StubAbortable(); - ExecutorService exec = Executors.newFixedThreadPool(1); - FakeTableFactory factory = new FakeTableFactory( - Collections. emptyMap()); - - // create a simple writer - IndexWriter writer = new IndexWriter(this.testName.getTableNameString(), abort, exec, factory); - writer.stop(this.testName.getTableNameString() + " finished"); - assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown); - assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown()); - } /** * With the move to using a pool of threads to write, we need to ensure that we still block until @@ -120,6 +80,7 @@ public void testSynchronouslyCompletesAllWrites() throws Exception { LOG.info("Starting " + testName.getTableNameString()); LOG.info("Current thread is interrupted: " + Thread.interrupted()); Abortable abort = new StubAbortable(); + Stoppable stop = Mockito.mock(Stoppable.class); ExecutorService exec = Executors.newFixedThreadPool(1); Map tables = new HashMap(); FakeTableFactory factory = new FakeTableFactory(tables); @@ -145,7 +106,12 @@ public Void answer(InvocationOnMock invocation) throws Throwable { // add the table to the set of tables, so its returned to the writer tables.put(new ImmutableBytesPtr(tableName), table); - IndexWriter writer = new IndexWriter(this.testName.getTableNameString(), abort, exec, factory); + // setup the writer and failure policy + ParallelWriterIndexCommitter committer = new ParallelWriterIndexCommitter(); + committer.setup(factory, exec, abort, stop); + KillServerOnFailurePolicy policy = new KillServerOnFailurePolicy(); + policy.setup(stop, abort); + IndexWriter writer = new IndexWriter(committer, policy); writer.write(indexUpdates); assertTrue("Writer returned before the table batch completed! Likely a race condition tripped", completed[0]); @@ -168,6 +134,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable { @Test public void testFailureOnRunningUpdateAbortsPending() throws Exception { Abortable abort = new StubAbortable(); + Stoppable stop = Mockito.mock(Stoppable.class); // single thread factory so the older request gets queued ExecutorService exec = Executors.newFixedThreadPool(1); Map tables = new HashMap(); @@ -213,7 +180,11 @@ public Void answer(InvocationOnMock invocation) throws Throwable { tables.put(new ImmutableBytesPtr(tableName), table); tables.put(new ImmutableBytesPtr(tableName2), table2); - IndexWriter writer = new IndexWriter(this.testName.getTableNameString(), abort, exec, factory); + ParallelWriterIndexCommitter committer = new ParallelWriterIndexCommitter(); + committer.setup(factory, exec, abort, stop); + KillServerOnFailurePolicy policy = new KillServerOnFailurePolicy(); + policy.setup(stop, abort); + IndexWriter writer = new IndexWriter(committer, policy); try { writer.write(indexUpdates); fail("Should not have successfully completed all index writes"); @@ -238,7 +209,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable { @SuppressWarnings("unchecked") @Test public void testShutdownInterruptsAsExpected() throws Exception { - + Stoppable stop = Mockito.mock(Stoppable.class); Abortable abort = new StubAbortable(); // single thread factory so the older request gets queued ExecutorService exec = Executors.newFixedThreadPool(1); @@ -269,8 +240,11 @@ public Void answer(InvocationOnMock invocation) throws Throwable { indexUpdates.add(new Pair(m, tableName)); // setup the writer - final IndexWriter writer = new IndexWriter(this.testName.getTableNameString(), abort, exec, - factory); + ParallelWriterIndexCommitter committer = new ParallelWriterIndexCommitter(); + committer.setup(factory, exec, abort, stop); + KillServerOnFailurePolicy policy = new KillServerOnFailurePolicy(); + policy.setup(stop, abort); + final IndexWriter writer = new IndexWriter(committer, policy); final boolean[] failedWrite = new boolean[] { false }; Thread primaryWriter = new Thread() { diff --git a/src/test/java/com/salesforce/hbase/index/write/TestParalleIndexWriter.java b/src/test/java/com/salesforce/hbase/index/write/TestParalleIndexWriter.java new file mode 100644 index 00000000..99ce10e6 --- /dev/null +++ b/src/test/java/com/salesforce/hbase/index/write/TestParalleIndexWriter.java @@ -0,0 +1,129 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index.write; + +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; +import com.salesforce.hbase.index.StubAbortable; +import com.salesforce.hbase.index.TableName; +import com.salesforce.hbase.index.table.HTableInterfaceReference; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; + +public class TestParalleIndexWriter { + + private static final Log LOG = LogFactory.getLog(TestParalleIndexWriter.class); + @Rule + public TableName test = new TableName(); + private final byte[] row = Bytes.toBytes("row"); + + @Test + public void testCorrectlyCleansUpResources() throws Exception{ + ExecutorService exec = Executors.newFixedThreadPool(1); + FakeTableFactory factory = new FakeTableFactory( + Collections. emptyMap()); + ParallelWriterIndexCommitter writer = new ParallelWriterIndexCommitter(); + Abortable mockAbort = Mockito.mock(Abortable.class); + Stoppable mockStop = Mockito.mock(Stoppable.class); + // create a simple writer + writer.setup(factory, exec, mockAbort, mockStop); + // stop the writer + writer.stop(this.test.getTableNameString() + " finished"); + assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown); + assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown()); + Mockito.verifyZeroInteractions(mockAbort, mockStop); + } + + @SuppressWarnings("unchecked") + @Test + public void testSynchronouslyCompletesAllWrites() throws Exception { + LOG.info("Starting " + test.getTableNameString()); + LOG.info("Current thread is interrupted: " + Thread.interrupted()); + Abortable abort = new StubAbortable(); + Stoppable stop = Mockito.mock(Stoppable.class); + ExecutorService exec = Executors.newFixedThreadPool(1); + Map tables = + new HashMap(); + FakeTableFactory factory = new FakeTableFactory(tables); + + ImmutableBytesPtr tableName = new ImmutableBytesPtr(this.test.getTableName()); + Put m = new Put(row); + m.add(Bytes.toBytes("family"), Bytes.toBytes("qual"), null); + Multimap indexUpdates = + ArrayListMultimap. create(); + indexUpdates.put(new HTableInterfaceReference(tableName), m); + + HTableInterface table = Mockito.mock(HTableInterface.class); + final boolean[] completed = new boolean[] { false }; + Mockito.when(table.batch(Mockito.anyList())).thenAnswer(new Answer() { + + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + // just keep track that it was called + completed[0] = true; + return null; + } + }); + Mockito.when(table.getTableName()).thenReturn(test.getTableName()); + // add the table to the set of tables, so its returned to the writer + tables.put(tableName, table); + + // setup the writer and failure policy + ParallelWriterIndexCommitter writer = new ParallelWriterIndexCommitter(); + writer.setup(factory, exec, abort, stop); + writer.write(indexUpdates); + assertTrue("Writer returned before the table batch completed! Likely a race condition tripped", + completed[0]); + writer.stop(this.test.getTableNameString() + " finished"); + assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown); + assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown()); + } +} From 75787655ea49d82f56965b576571482888773654 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Tue, 24 Sep 2013 17:09:11 -0700 Subject: [PATCH 069/102] Fix explain plan to reflect hinting that foces a range scan --- pom.xml | 2 +- .../phoenix/compile/ScanRanges.java | 19 ++++++--- .../phoenix/compile/WhereOptimizer.java | 33 ++++++++++++--- .../phoenix/index/PhoenixIndexBuilder.java | 4 +- .../phoenix/index/PhoenixIndexCodec.java | 1 - .../query/ConnectionQueryServicesImpl.java | 3 +- .../ConnectionlessQueryServicesImpl.java | 23 ++++++++++- .../phoenix/schema/SaltingUtil.java | 35 ---------------- .../com/salesforce/phoenix/util/ScanUtil.java | 3 +- .../salesforce/phoenix/util/SchemaUtil.java | 38 ++++++++++++++++++ .../regionserver/wal/IndexedWALEdit.java | 2 + .../StatementHintsCompilationTest.java | 40 ++++++++++++++----- 12 files changed, 136 insertions(+), 67 deletions(-) diff --git a/pom.xml b/pom.xml index 00e39b06..07316e4a 100644 --- a/pom.xml +++ b/pom.xml @@ -63,7 +63,7 @@ true - 0.94.10 + 0.94.12 1.2 1.0.4 0.11.0 diff --git a/src/main/java/com/salesforce/phoenix/compile/ScanRanges.java b/src/main/java/com/salesforce/phoenix/compile/ScanRanges.java index 66356134..b8ec0c9a 100644 --- a/src/main/java/com/salesforce/phoenix/compile/ScanRanges.java +++ b/src/main/java/com/salesforce/phoenix/compile/ScanRanges.java @@ -42,28 +42,34 @@ public class ScanRanges { private static final List> EVERYTHING_RANGES = Collections.>emptyList(); private static final List> NOTHING_RANGES = Collections.>singletonList(Collections.singletonList(KeyRange.EMPTY_RANGE)); - public static final ScanRanges EVERYTHING = new ScanRanges(EVERYTHING_RANGES,null); - public static final ScanRanges NOTHING = new ScanRanges(NOTHING_RANGES,null); + public static final ScanRanges EVERYTHING = new ScanRanges(EVERYTHING_RANGES,null,false); + public static final ScanRanges NOTHING = new ScanRanges(NOTHING_RANGES,null,false); public static ScanRanges create(List> ranges, RowKeySchema schema) { + return create(ranges, schema, false); + } + + public static ScanRanges create(List> ranges, RowKeySchema schema, boolean forceRangeScan) { if (ranges.isEmpty()) { return EVERYTHING; } else if (ranges.size() == 1 && ranges.get(0).size() == 1 && ranges.get(0).get(0) == KeyRange.EMPTY_RANGE) { return NOTHING; } - return new ScanRanges(ranges, schema); + return new ScanRanges(ranges, schema, forceRangeScan); } private SkipScanFilter filter; private final List> ranges; private final RowKeySchema schema; + private final boolean forceRangeScan; - private ScanRanges (List> ranges, RowKeySchema schema) { + private ScanRanges (List> ranges, RowKeySchema schema, boolean forceRangeScan) { this.ranges = ranges; this.schema = schema; - if (schema != null && !ranges.isEmpty() ) { + if (schema != null && !ranges.isEmpty()) { this.filter = new SkipScanFilter(ranges, schema); } + this.forceRangeScan = forceRangeScan; } public SkipScanFilter getSkipScanFilter() { @@ -93,6 +99,9 @@ public boolean isDegenerate() { * not the last key slot */ public boolean useSkipScanFilter() { + if (forceRangeScan) { + return false; + } boolean hasRangeKey = false, useSkipScan = false; for (List orRanges : ranges) { useSkipScan |= orRanges.size() > 1 | hasRangeKey; diff --git a/src/main/java/com/salesforce/phoenix/compile/WhereOptimizer.java b/src/main/java/com/salesforce/phoenix/compile/WhereOptimizer.java index 202c0057..96bde84d 100644 --- a/src/main/java/com/salesforce/phoenix/compile/WhereOptimizer.java +++ b/src/main/java/com/salesforce/phoenix/compile/WhereOptimizer.java @@ -27,22 +27,43 @@ ******************************************************************************/ package com.salesforce.phoenix.compile; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import com.google.common.base.Preconditions; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; -import com.salesforce.phoenix.expression.*; +import com.salesforce.phoenix.expression.AndExpression; +import com.salesforce.phoenix.expression.ComparisonExpression; +import com.salesforce.phoenix.expression.Expression; +import com.salesforce.phoenix.expression.InListExpression; +import com.salesforce.phoenix.expression.IsNullExpression; +import com.salesforce.phoenix.expression.LikeExpression; +import com.salesforce.phoenix.expression.LiteralExpression; +import com.salesforce.phoenix.expression.OrExpression; +import com.salesforce.phoenix.expression.RowKeyColumnExpression; import com.salesforce.phoenix.expression.function.ScalarFunction; import com.salesforce.phoenix.expression.visitor.TraverseNoExpressionVisitor; import com.salesforce.phoenix.parse.FilterableStatement; import com.salesforce.phoenix.parse.HintNode.Hint; import com.salesforce.phoenix.query.KeyRange; import com.salesforce.phoenix.query.QueryConstants; -import com.salesforce.phoenix.schema.*; -import com.salesforce.phoenix.util.*; +import com.salesforce.phoenix.schema.ColumnModifier; +import com.salesforce.phoenix.schema.PColumn; +import com.salesforce.phoenix.schema.PDataType; +import com.salesforce.phoenix.schema.PTable; +import com.salesforce.phoenix.schema.RowKeySchema; +import com.salesforce.phoenix.schema.SaltingUtil; +import com.salesforce.phoenix.util.ByteUtil; +import com.salesforce.phoenix.util.ScanUtil; +import com.salesforce.phoenix.util.SchemaUtil; /** @@ -157,13 +178,13 @@ public static Expression pushKeyExpressionsToScan(StatementContext context, Filt if (ScanUtil.isAllSingleRowScan(cnf, table.getRowKeySchema())) { cnf.addFirst(SALT_PLACEHOLDER); ranges = SaltingUtil.flattenRanges(cnf, table.getRowKeySchema(), table.getBucketNum()); - schema = SaltingUtil.VAR_BINARY_SCHEMA; + schema = SchemaUtil.VAR_BINARY_SCHEMA; } else { cnf.addFirst(SaltingUtil.generateAllSaltingRanges(table.getBucketNum())); } } } - context.setScanRanges(ScanRanges.create(ranges, schema)); + context.setScanRanges(ScanRanges.create(ranges, schema, statement.getHint().hasHint(Hint.RANGE_SCAN))); return whereClause.accept(new RemoveExtractedNodesVisitor(extractNodes)); } diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java index 20922a9c..ced63eaa 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java @@ -35,7 +35,7 @@ import com.salesforce.phoenix.compile.ScanRanges; import com.salesforce.phoenix.query.KeyRange; import com.salesforce.phoenix.schema.PDataType; -import com.salesforce.phoenix.schema.SaltingUtil; +import com.salesforce.phoenix.util.SchemaUtil; /** * Index builder for covered-columns index that ties into phoenix for faster use. @@ -54,7 +54,7 @@ public void batchStarted(MiniBatchOperationInProgress> m keys.add(PDataType.VARBINARY.getKeyRange(m.getRow())); maintainers.addAll(getCodec().getIndexMaintainers(m.getAttributesMap())); } - ScanRanges scanRanges = ScanRanges.create(Collections.singletonList(keys), SaltingUtil.VAR_BINARY_SCHEMA); + ScanRanges scanRanges = ScanRanges.create(Collections.singletonList(keys), SchemaUtil.VAR_BINARY_SCHEMA); Scan scan = new Scan(); scan.setRaw(true); // Project into scan only the columns we need to build the new index row and diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java index d11d7482..457992a4 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexCodec.java @@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import com.google.common.collect.Lists; diff --git a/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java b/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java index c484ffd8..d82697de 100644 --- a/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java +++ b/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java @@ -107,7 +107,6 @@ import com.salesforce.phoenix.schema.PTable; import com.salesforce.phoenix.schema.PTableType; import com.salesforce.phoenix.schema.ReadOnlyTableException; -import com.salesforce.phoenix.schema.SaltingUtil; import com.salesforce.phoenix.schema.TableAlreadyExistsException; import com.salesforce.phoenix.schema.TableNotFoundException; import com.salesforce.phoenix.util.ByteUtil; @@ -856,7 +855,7 @@ private void upgradeTablesFrom2_0to2_1(HBaseAdmin admin, HTableDescriptor newDes indexRowsToUpdate.add(keyRange); } } - ScanRanges ranges = ScanRanges.create(Collections.singletonList(indexRowsToUpdate), SaltingUtil.VAR_BINARY_SCHEMA); + ScanRanges ranges = ScanRanges.create(Collections.singletonList(indexRowsToUpdate), SchemaUtil.VAR_BINARY_SCHEMA); Scan indexScan = new Scan(); scan.addFamily(TABLE_FAMILY_BYTES); indexScan.setFilter(ranges.getSkipScanFilter()); diff --git a/src/main/java/com/salesforce/phoenix/query/ConnectionlessQueryServicesImpl.java b/src/main/java/com/salesforce/phoenix/query/ConnectionlessQueryServicesImpl.java index 49e6891c..7aafbb53 100644 --- a/src/main/java/com/salesforce/phoenix/query/ConnectionlessQueryServicesImpl.java +++ b/src/main/java/com/salesforce/phoenix/query/ConnectionlessQueryServicesImpl.java @@ -30,10 +30,13 @@ import static com.salesforce.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES; import java.sql.SQLException; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Properties; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; @@ -58,6 +61,7 @@ import com.salesforce.phoenix.schema.PTable; import com.salesforce.phoenix.schema.PTableImpl; import com.salesforce.phoenix.schema.PTableType; +import com.salesforce.phoenix.schema.TableRef; import com.salesforce.phoenix.util.PhoenixRuntime; import com.salesforce.phoenix.util.SchemaUtil; @@ -90,12 +94,27 @@ public HTableInterface getTable(byte[] tableName) throws SQLException { @Override public StatsManager getStatsManager() { - throw new UnsupportedOperationException(); + return new StatsManager() { + + @Override + public byte[] getMinKey(TableRef table) { + return HConstants.EMPTY_START_ROW; + } + + @Override + public byte[] getMaxKey(TableRef table) { + return HConstants.EMPTY_END_ROW; + } + + @Override + public void updateStats(TableRef table) throws SQLException { + } + }; } @Override public List getAllTableRegions(byte[] tableName) throws SQLException { - throw new UnsupportedOperationException(); + return Collections.singletonList(new HRegionLocation(new HRegionInfo(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW),"localhost",-1)); } @Override diff --git a/src/main/java/com/salesforce/phoenix/schema/SaltingUtil.java b/src/main/java/com/salesforce/phoenix/schema/SaltingUtil.java index 4e4944aa..a6bdf5a6 100644 --- a/src/main/java/com/salesforce/phoenix/schema/SaltingUtil.java +++ b/src/main/java/com/salesforce/phoenix/schema/SaltingUtil.java @@ -37,7 +37,6 @@ import com.salesforce.phoenix.compile.ScanRanges; import com.salesforce.phoenix.query.KeyRange; import com.salesforce.phoenix.query.KeyRange.Bound; -import com.salesforce.phoenix.schema.RowKeySchema.RowKeySchemaBuilder; import com.salesforce.phoenix.util.ScanUtil; @@ -45,40 +44,6 @@ * Utility methods related to transparent salting of row keys. */ public class SaltingUtil { - public static RowKeySchema VAR_BINARY_SCHEMA = new RowKeySchemaBuilder(1).addField(new PDatum() { - - @Override - public boolean isNullable() { - return false; - } - - @Override - public PDataType getDataType() { - return PDataType.VARBINARY; - } - - @Override - public Integer getByteSize() { - return null; - } - - @Override - public Integer getMaxLength() { - return null; - } - - @Override - public Integer getScale() { - return null; - } - - @Override - public ColumnModifier getColumnModifier() { - return null; - } - - }, false, null).build(); - public static final int NUM_SALTING_BYTES = 1; public static final Integer MAX_BUCKET_NUM = 256; // Unsigned byte. public static final String SALTING_COLUMN_NAME = "_SALT"; diff --git a/src/main/java/com/salesforce/phoenix/util/ScanUtil.java b/src/main/java/com/salesforce/phoenix/util/ScanUtil.java index 156b6257..d872f7b0 100644 --- a/src/main/java/com/salesforce/phoenix/util/ScanUtil.java +++ b/src/main/java/com/salesforce/phoenix/util/ScanUtil.java @@ -50,7 +50,6 @@ import com.salesforce.phoenix.query.QueryConstants; import com.salesforce.phoenix.schema.PDataType; import com.salesforce.phoenix.schema.RowKeySchema; -import com.salesforce.phoenix.schema.SaltingUtil; /** @@ -398,7 +397,7 @@ public static ScanRanges newScanRanges(List mutations) throws SQLExcep for (Mutation m : mutations) { keys.add(PDataType.VARBINARY.getKeyRange(m.getRow())); } - ScanRanges keyRanges = ScanRanges.create(Collections.singletonList(keys), SaltingUtil.VAR_BINARY_SCHEMA); + ScanRanges keyRanges = ScanRanges.create(Collections.singletonList(keys), SchemaUtil.VAR_BINARY_SCHEMA); return keyRanges; } } diff --git a/src/main/java/com/salesforce/phoenix/util/SchemaUtil.java b/src/main/java/com/salesforce/phoenix/util/SchemaUtil.java index 76a038a1..6f55d486 100644 --- a/src/main/java/com/salesforce/phoenix/util/SchemaUtil.java +++ b/src/main/java/com/salesforce/phoenix/util/SchemaUtil.java @@ -88,14 +88,18 @@ import com.salesforce.phoenix.query.QueryConstants; import com.salesforce.phoenix.schema.AmbiguousColumnException; import com.salesforce.phoenix.schema.ColumnFamilyNotFoundException; +import com.salesforce.phoenix.schema.ColumnModifier; import com.salesforce.phoenix.schema.ColumnNotFoundException; import com.salesforce.phoenix.schema.PColumn; import com.salesforce.phoenix.schema.PColumnFamily; import com.salesforce.phoenix.schema.PDataType; +import com.salesforce.phoenix.schema.PDatum; import com.salesforce.phoenix.schema.PMetaData; import com.salesforce.phoenix.schema.PName; import com.salesforce.phoenix.schema.PTable; import com.salesforce.phoenix.schema.PTableType; +import com.salesforce.phoenix.schema.RowKeySchema; +import com.salesforce.phoenix.schema.RowKeySchema.RowKeySchemaBuilder; import com.salesforce.phoenix.schema.SaltingUtil; @@ -112,6 +116,40 @@ public class SchemaUtil { private static final int VAR_LENGTH_ESTIMATE = 10; public static final DataBlockEncoding DEFAULT_DATA_BLOCK_ENCODING = DataBlockEncoding.FAST_DIFF; + public static RowKeySchema VAR_BINARY_SCHEMA = new RowKeySchemaBuilder(1).addField(new PDatum() { + + @Override + public boolean isNullable() { + return false; + } + + @Override + public PDataType getDataType() { + return PDataType.VARBINARY; + } + + @Override + public Integer getByteSize() { + return null; + } + + @Override + public Integer getMaxLength() { + return null; + } + + @Override + public Integer getScale() { + return null; + } + + @Override + public ColumnModifier getColumnModifier() { + return null; + } + + }, false, null).build(); + /** * May not be instantiated */ diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/IndexedWALEdit.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/IndexedWALEdit.java index 598de8c4..e7e2cb53 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/IndexedWALEdit.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/IndexedWALEdit.java @@ -27,6 +27,7 @@ public class IndexedWALEdit extends WALEdit { * Copy-constructor. Only does a surface copy of the delegates fields - no actual data is copied, only referenced. * @param delegate to copy */ + @SuppressWarnings("deprecation") public IndexedWALEdit(WALEdit delegate) { this.delegate = delegate; // reset the delegate's fields @@ -46,6 +47,7 @@ public void setCompressionContext(CompressionContext context) { "Compression not supported for IndexedWALEdit! If you are using HBase 0.94.9+, use IndexedWALEditCodec instead."); } + @SuppressWarnings("deprecation") @Override public void readFields(DataInput in) throws IOException { delegate.getKeyValues().clear(); diff --git a/src/test/java/com/salesforce/phoenix/compile/StatementHintsCompilationTest.java b/src/test/java/com/salesforce/phoenix/compile/StatementHintsCompilationTest.java index 105f5eba..1ba29e9c 100644 --- a/src/test/java/com/salesforce/phoenix/compile/StatementHintsCompilationTest.java +++ b/src/test/java/com/salesforce/phoenix/compile/StatementHintsCompilationTest.java @@ -32,7 +32,9 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.sql.Connection; import java.sql.DriverManager; +import java.sql.ResultSet; import java.sql.SQLException; import java.util.Collections; import java.util.List; @@ -50,6 +52,7 @@ import com.salesforce.phoenix.parse.SQLParser; import com.salesforce.phoenix.parse.SelectStatement; import com.salesforce.phoenix.query.BaseConnectionlessQueryTest; +import com.salesforce.phoenix.util.QueryUtil; /** @@ -60,6 +63,20 @@ public class StatementHintsCompilationTest extends BaseConnectionlessQueryTest { private static StatementContext compileStatement(String query, Scan scan, List binds) throws SQLException { return compileStatement(query, scan, binds, null, null); } + + private static boolean usingSkipScan(Scan scan) { + Filter filter = scan.getFilter(); + if (filter instanceof FilterList) { + FilterList filterList = (FilterList) filter; + for (Filter childFilter : filterList.getFilters()) { + if (childFilter instanceof SkipScanFilter) { + return true; + } + } + return false; + } + return filter instanceof SkipScanFilter; + } private static StatementContext compileStatement(String query, Scan scan, List binds, Integer limit, Set extractedNodes) throws SQLException { SQLParser parser = new SQLParser(query); @@ -86,12 +103,7 @@ public void testSelectForceSkipScan() throws Exception { List binds = Collections.emptyList(); compileStatement(query, scan, binds); - // Forced to using skip scan filter. - Filter filter = scan.getFilter(); - if (filter instanceof FilterList) { - filter = ((FilterList) filter).getFilters().get(0); - } - assertTrue("The first filter should be SkipScanFilter.", filter instanceof SkipScanFilter); + assertTrue("The first filter should be SkipScanFilter.", usingSkipScan(scan)); } @Test @@ -103,10 +115,16 @@ public void testSelectForceRangeScan() throws Exception { compileStatement(query, scan, binds); // Verify that it is not using SkipScanFilter. - Filter filter = scan.getFilter(); - if (filter instanceof FilterList) { - filter = ((FilterList) filter).getFilters().get(0); - } - assertFalse("The first filter should not be SkipScanFilter.", filter instanceof SkipScanFilter); + assertFalse("The first filter should not be SkipScanFilter.", usingSkipScan(scan)); + } + + @Test + public void testSelectForceRangeScanForEH() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + conn.createStatement().execute("create table eh (organization_id char(15) not null,parent_id char(15) not null, created_date date not null, entity_history_id char(15) not null constraint pk primary key (organization_id, parent_id, created_date, entity_history_id))"); + ResultSet rs = conn.createStatement().executeQuery("explain select /*+ RANGE_SCAN */ ORGANIZATION_ID, PARENT_ID, CREATED_DATE, ENTITY_HISTORY_ID from eh where ORGANIZATION_ID='111111111111111' and SUBSTR(PARENT_ID, 1, 3) = 'foo' and TO_DATE ('2012-0-1 00:00:00') <= CREATED_DATE and CREATED_DATE <= TO_DATE ('2012-11-31 00:00:00') order by ORGANIZATION_ID, PARENT_ID, CREATED_DATE DESC, ENTITY_HISTORY_ID limit 100"); + assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER EH '111111111111111',['foo'-'fop'),['2011-12-01 00:00:00.000'-'2012-12-01 00:00:00.000']\n" + + " SERVER TOP 100 ROWS SORTED BY [ORGANIZATION_ID, PARENT_ID, CREATED_DATE DESC, ENTITY_HISTORY_ID]\n" + + "CLIENT MERGE SORT",QueryUtil.getExplainPlan(rs)); } } From c042596d653545692695222138ee5cf7c8c1e93d Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Tue, 24 Sep 2013 17:39:14 -0700 Subject: [PATCH 070/102] Bug fix for IndexWriter. Configuration keys for the IndexCommitter and FailurePolicy weren't specified, causing a null pointer. Adding tests to catch this case in the future. --- .../hbase/index/write/IndexWriter.java | 9 +++++---- .../hbase/index/write/TestIndexWriter.java | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/salesforce/hbase/index/write/IndexWriter.java b/src/main/java/com/salesforce/hbase/index/write/IndexWriter.java index e1b1d0a6..1368aecb 100644 --- a/src/main/java/com/salesforce/hbase/index/write/IndexWriter.java +++ b/src/main/java/com/salesforce/hbase/index/write/IndexWriter.java @@ -57,7 +57,8 @@ public class IndexWriter implements Stoppable { private static final Log LOG = LogFactory.getLog(IndexWriter.class); - private static final String INDEX_COMMITTER_CONF_KEY = null; + private static final String INDEX_COMMITTER_CONF_KEY = "index.writer.commiter.class"; + private static final String INDEX_FAILURE_POLICY_CONF_KEY = "index.writer.failurepolicy.class"; private AtomicBoolean stopped = new AtomicBoolean(false); private IndexCommitter writer; private IndexFailurePolicy failurePolicy; @@ -72,7 +73,7 @@ public IndexWriter(RegionCoprocessorEnvironment env) throws IOException { this.failurePolicy.setup(this, env); } - private static IndexCommitter getCommitter(RegionCoprocessorEnvironment env) throws IOException { + static IndexCommitter getCommitter(RegionCoprocessorEnvironment env) throws IOException { Configuration conf = env.getConfiguration(); try { IndexCommitter committer = @@ -86,12 +87,12 @@ private static IndexCommitter getCommitter(RegionCoprocessorEnvironment env) thr } } - private static IndexFailurePolicy getFailurePolicy(RegionCoprocessorEnvironment env) + static IndexFailurePolicy getFailurePolicy(RegionCoprocessorEnvironment env) throws IOException { Configuration conf = env.getConfiguration(); try { IndexFailurePolicy committer = - conf.getClass(INDEX_COMMITTER_CONF_KEY, KillServerOnFailurePolicy.class, + conf.getClass(INDEX_FAILURE_POLICY_CONF_KEY, KillServerOnFailurePolicy.class, IndexFailurePolicy.class).newInstance(); return committer; } catch (InstantiationException e) { diff --git a/src/test/java/com/salesforce/hbase/index/write/TestIndexWriter.java b/src/test/java/com/salesforce/hbase/index/write/TestIndexWriter.java index 94d5b284..e523b6e0 100644 --- a/src/test/java/com/salesforce/hbase/index/write/TestIndexWriter.java +++ b/src/test/java/com/salesforce/hbase/index/write/TestIndexWriter.java @@ -28,6 +28,7 @@ package com.salesforce.hbase.index.write; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -44,11 +45,13 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.junit.Rule; @@ -68,6 +71,21 @@ public class TestIndexWriter { public TableName testName = new TableName(); private final byte[] row = Bytes.toBytes("row"); + @Test + public void getDefaultWriter() throws Exception { + Configuration conf = new Configuration(false); + RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class); + Mockito.when(env.getConfiguration()).thenReturn(conf); + assertNotNull(IndexWriter.getCommitter(env)); + } + + @Test + public void getDefaultFailurePolicy() throws Exception { + Configuration conf = new Configuration(false); + RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class); + Mockito.when(env.getConfiguration()).thenReturn(conf); + assertNotNull(IndexWriter.getFailurePolicy(env)); + } /** * With the move to using a pool of threads to write, we need to ensure that we still block until From 084694c0d3df3c49d89b1539a847cffcb0eebd14 Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Tue, 24 Sep 2013 17:40:03 -0700 Subject: [PATCH 071/102] Cleanup to IndexUpdateManager and HTableInterfaceReference Former: better toString format Latter: fixing javadoc --- .../covered/update/IndexUpdateManager.java | 2 +- .../index/table/HTableInterfaceReference.java | 18 ++---------------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java b/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java index e781448d..53350a46 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java +++ b/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java @@ -232,7 +232,7 @@ public String toString() { + ((m instanceof Put) ? m.getTimeStamp() + " " : "")); sb.append("\n"); if (m.getFamilyMap().isEmpty()) { - sb.append("=== EMPTY ==="); + sb.append("\t\t=== EMPTY ===\n"); } for (List kvs : m.getFamilyMap().values()) { for (KeyValue kv : kvs) { diff --git a/src/main/java/com/salesforce/hbase/index/table/HTableInterfaceReference.java b/src/main/java/com/salesforce/hbase/index/table/HTableInterfaceReference.java index f6ad2029..0bce2217 100644 --- a/src/main/java/com/salesforce/hbase/index/table/HTableInterfaceReference.java +++ b/src/main/java/com/salesforce/hbase/index/table/HTableInterfaceReference.java @@ -1,26 +1,12 @@ package com.salesforce.hbase.index.table; -import org.apache.hadoop.hbase.CoprocessorEnvironment; -import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.util.Bytes; import com.salesforce.hbase.index.util.ImmutableBytesPtr; /** - * Reference to an HTableInterface that only gets the underlying {@link HTableInterface} from the - * {@link CoprocessorEnvironment} on calls to {@link #getTable()}. Until calling {@link #getTable()} - * , this class just contains the name of the table and an optional {@link HTableFactory}. After - * calling {@link #getTable()}, this holds a reference to that {@link HTableInterface}, - * even if that table is closed. - *

- * Whenever calling {@link #getTable()}, an internal reference counter is incremented. Similarly, - * the reference count is decremented by calling {@link #close()}. The underlying table, if - * previously resolved, will be closed on calls to {@link #close()} only if the underlying reference - * count is zero. - *

- * This class is not thread-safe when resolving the reference to the {@link HTableInterface} - - * multi-threaded usage must employ external locking to ensure that multiple {@link HTableInterface} - * s are not resolved. + * Reference to an HTableInterface. Currently, its pretty simple in that it is just a wrapper around + * the table name. */ public class HTableInterfaceReference { From 2481825c0422132ea3458862a66868cce12f56e9 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Tue, 24 Sep 2013 21:03:40 -0700 Subject: [PATCH 072/102] Fix build warnings --- src/main/java/com/salesforce/hbase/index/Indexer.java | 1 + .../salesforce/hbase/index/write/TestParalleIndexWriter.java | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/com/salesforce/hbase/index/Indexer.java b/src/main/java/com/salesforce/hbase/index/Indexer.java index 4b000018..b807d7ad 100644 --- a/src/main/java/com/salesforce/hbase/index/Indexer.java +++ b/src/main/java/com/salesforce/hbase/index/Indexer.java @@ -266,6 +266,7 @@ public MultiMutation(ImmutableBytesPtr rowkey, boolean writeToWal) { /** * @param stored */ + @SuppressWarnings("deprecation") public void addAll(Mutation stored) { // add all the kvs for (Entry> kvs : stored.getFamilyMap().entrySet()) { diff --git a/src/test/java/com/salesforce/hbase/index/write/TestParalleIndexWriter.java b/src/test/java/com/salesforce/hbase/index/write/TestParalleIndexWriter.java index 99ce10e6..3937e34d 100644 --- a/src/test/java/com/salesforce/hbase/index/write/TestParalleIndexWriter.java +++ b/src/test/java/com/salesforce/hbase/index/write/TestParalleIndexWriter.java @@ -29,7 +29,6 @@ import static org.junit.Assert.assertTrue; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -44,7 +43,6 @@ import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Pair; import org.junit.Rule; import org.junit.Test; import org.mockito.Mockito; From 5a250d5b20c6d42a975cfee9258da96008516eed Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Wed, 25 Sep 2013 12:45:34 -0700 Subject: [PATCH 073/102] Bug fix for table caching + cleanup of tostrings. Caching was closing HTables on shutdown, but the CoprocessorHost was also closing the same tables, causing the concurrentModificationException on shutdown sometimes, which would cause the test to hang. Also, using a SoftReference for the HTable doesn't work b/c the coprocessorHost has a strong reference to the tables, causing them to never be cleaned up. Solution was to switch to a standard LRU cache for the HTables; fortunately, also meant much simpler code. Also, this meant fixing up some tests (and adding a new one for the caching factory). Further, adding a rowkey tosString to IndexUpdateManager so we can better visualize the updates being made. --- .../covered/update/IndexUpdateManager.java | 1 + .../index/table/CachingHTableFactory.java | 114 +++++++--------- .../index/table/CoprocessorHTableFactory.java | 26 +--- .../write/ParallelWriterIndexCommitter.java | 7 +- .../index/write/TestCachingHTableFactory.java | 68 ++++++++++ .../hbase/index/write/TestIndexWriter.java | 6 +- .../TestParalleWriterIndexCommitter.java | 127 ++++++++++++++++++ 7 files changed, 259 insertions(+), 90 deletions(-) create mode 100644 src/test/java/com/salesforce/hbase/index/write/TestCachingHTableFactory.java create mode 100644 src/test/java/com/salesforce/hbase/index/write/TestParalleWriterIndexCommitter.java diff --git a/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java b/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java index 53350a46..1a9d8c6f 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java +++ b/src/main/java/com/salesforce/hbase/index/covered/update/IndexUpdateManager.java @@ -230,6 +230,7 @@ public String toString() { } sb.append(m.getClass().getSimpleName() + ":" + ((m instanceof Put) ? m.getTimeStamp() + " " : "")); + sb.append(" row=" + Bytes.toString(m.getRow())); sb.append("\n"); if (m.getFamilyMap().isEmpty()) { sb.append("\t\t=== EMPTY ===\n"); diff --git a/src/main/java/com/salesforce/hbase/index/table/CachingHTableFactory.java b/src/main/java/com/salesforce/hbase/index/table/CachingHTableFactory.java index f20520bb..8d7eb730 100644 --- a/src/main/java/com/salesforce/hbase/index/table/CachingHTableFactory.java +++ b/src/main/java/com/salesforce/hbase/index/table/CachingHTableFactory.java @@ -28,13 +28,12 @@ package com.salesforce.hbase.index.table; import java.io.IOException; -import java.lang.ref.ReferenceQueue; -import java.lang.ref.SoftReference; -import java.util.HashMap; import java.util.Map; +import org.apache.commons.collections.map.LRUMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.util.Bytes; @@ -42,8 +41,8 @@ /** * A simple cache that just uses usual GC mechanisms to cleanup unused {@link HTableInterface}s. - * When requesting an {@link HTableInterface} via {@link #getTable(byte[])}, you may get the same - * table as last time, or it may be a new table. + * When requesting an {@link HTableInterface} via {@link #getTable}, you may get the same table as + * last time, or it may be a new table. *

* You should not call {@link HTableInterface#close()} that is handled when the table goes * out of scope. Along the same lines, you must ensure to not keep a reference to the table for @@ -51,36 +50,64 @@ */ public class CachingHTableFactory implements HTableFactory { + /** + * LRUMap that closes the {@link HTableInterface} when the table is evicted + */ + @SuppressWarnings("serial") + public class HTableInterfaceLRUMap extends LRUMap { + + public HTableInterfaceLRUMap(int cacheSize) { + super(cacheSize); + } + + @Override + protected boolean removeLRU(LinkEntry entry) { + HTableInterface table = (HTableInterface) entry.getValue(); + if (LOG.isDebugEnabled()) { + LOG.debug("Closing connection to table: " + Bytes.toString(table.getTableName()) + + " because it was evicted from the cache."); + } + try { + table.close(); + } catch (IOException e) { + LOG.info("Failed to correctly close HTable: " + Bytes.toString(table.getTableName()) + + " ignoring since being removed from queue."); + } + return true; + } + } + + public static int getCacheSize(Configuration conf) { + return conf.getInt(CACHE_SIZE_KEY, DEFAULT_CACHE_SIZE); + } + private static final Log LOG = LogFactory.getLog(CachingHTableFactory.class); - private HTableFactory delegate; + private static final String CACHE_SIZE_KEY = "index.tablefactory.cache.size"; + private static final int DEFAULT_CACHE_SIZE = 10; - Map> openTables = - new HashMap>(); + private HTableFactory delegate; - private ReferenceQueue referenceQueue; + @SuppressWarnings("rawtypes") + Map openTables; - private Thread cleanupThread; + public CachingHTableFactory(HTableFactory tableFactory, Configuration conf) { + this(tableFactory, getCacheSize(conf)); + } - public CachingHTableFactory(HTableFactory tableFactory) { - this.delegate = tableFactory; - this.referenceQueue = new ReferenceQueue(); - this.cleanupThread = new Thread(new TableCleaner(referenceQueue), "cached-table-cleanup"); - cleanupThread.setDaemon(true); - cleanupThread.start(); + public CachingHTableFactory(HTableFactory factory, int cacheSize) { + this.delegate = factory; + openTables = new HTableInterfaceLRUMap(cacheSize); } @Override + @SuppressWarnings("unchecked") public HTableInterface getTable(ImmutableBytesPtr tablename) throws IOException { ImmutableBytesPtr tableBytes = new ImmutableBytesPtr(tablename); synchronized (openTables) { - SoftReference ref = openTables.get(tableBytes); - // the reference may be null, in which case this is a new table, or the underlying HTable may - // have been GC'ed. - @SuppressWarnings("resource") - HTableInterface table = ref == null ? null : ref.get(); + HTableInterface table = (HTableInterface) openTables.get(tableBytes); if (table == null) { table = delegate.getTable(tablename); - openTables.put(tableBytes, new SoftReference(table, referenceQueue)); + openTables.put(tableBytes, table); } return table; } @@ -88,49 +115,6 @@ public HTableInterface getTable(ImmutableBytesPtr tablename) throws IOException @Override public void shutdown() { - this.cleanupThread.interrupt(); this.delegate.shutdown(); } - - /** - * Cleaner to ensure that any tables that are GC'ed are also closed. - */ - private class TableCleaner implements Runnable { - - private ReferenceQueue queue; - - public TableCleaner(ReferenceQueue referenceQueue) { - this.queue = referenceQueue; - } - - @Override - public void run() { - try { - HTableInterface table = this.queue.remove().get(); - if (table != null) { - try { - table.close(); - } catch (IOException e) { - LOG.error( - "Failed to correctly close htable, ignoring it further since it is being GC'ed", e); - } - } - } catch (InterruptedException e) { - LOG.info("Recieved an interrupt - assuming system is going down. Closing all remaining HTables and quitting!"); - for (SoftReference ref : openTables.values()) { - HTableInterface table = ref.get(); - if (table != null) { - try { - LOG.info("Closing connection to index table: " + Bytes.toString(table.getTableName())); - table.close(); - } catch (IOException ioe) { - LOG.error( - "Failed to correctly close htable on shutdown! Ignoring and closing remaining tables", - ioe); - } - } - } - } - } - } } \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/table/CoprocessorHTableFactory.java b/src/main/java/com/salesforce/hbase/index/table/CoprocessorHTableFactory.java index b017ac8b..396bd249 100644 --- a/src/main/java/com/salesforce/hbase/index/table/CoprocessorHTableFactory.java +++ b/src/main/java/com/salesforce/hbase/index/table/CoprocessorHTableFactory.java @@ -2,15 +2,19 @@ import java.io.IOException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.util.Bytes; import com.salesforce.hbase.index.util.ImmutableBytesPtr; public class CoprocessorHTableFactory implements HTableFactory { + private static final Log LOG = LogFactory.getLog(CoprocessorHTableFactory.class); private CoprocessorEnvironment e; public CoprocessorHTableFactory(CoprocessorEnvironment e) { @@ -28,26 +32,10 @@ public HTableInterface getTable(ImmutableBytesPtr tablename) throws IOException conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 30000); conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 5000); - return this.e.getTable(copyOrBytesIfExact(tablename)); - } - - /** - * There are possible outcomes: a copy of the section of bytes that we care about, or the exact - * byte array, if the {@link ImmutableBytesPtr} doesn't map 'into' the byte array. This saves us - * doing an extra byte copy for each table name. - *

- * Either way, you should not modify the returned bytes - it should be assumed that they are - * backing the {@link ImmutableBytesPtr}. - * @param bytes to introspect - * @return a byte[] from the {@link ImmutableBytesPtr} - */ - private byte[] copyOrBytesIfExact(ImmutableBytesPtr bytes) { - if (bytes.getOffset() == 0) { - if (bytes.getLength() == bytes.get().length) { - return bytes.get(); - } + if (LOG.isDebugEnabled()) { + LOG.debug("Getting access to new HTable: " + Bytes.toString(tablename.copyBytesIfNecessary())); } - return bytes.copyBytes(); + return this.e.getTable(tablename.copyBytesIfNecessary()); } @Override diff --git a/src/main/java/com/salesforce/hbase/index/write/ParallelWriterIndexCommitter.java b/src/main/java/com/salesforce/hbase/index/write/ParallelWriterIndexCommitter.java index 72482ea2..dddd7b55 100644 --- a/src/main/java/com/salesforce/hbase/index/write/ParallelWriterIndexCommitter.java +++ b/src/main/java/com/salesforce/hbase/index/write/ParallelWriterIndexCommitter.java @@ -85,7 +85,7 @@ public class ParallelWriterIndexCommitter implements IndexCommitter { public void setup(IndexWriter parent, RegionCoprocessorEnvironment env) { Configuration conf = env.getConfiguration(); setup(getDefaultDelegateHTableFactory(env), getDefaultExecutor(conf), - env.getRegionServerServices(), parent); + env.getRegionServerServices(), parent, CachingHTableFactory.getCacheSize(conf)); } /** @@ -93,9 +93,10 @@ public void setup(IndexWriter parent, RegionCoprocessorEnvironment env) { *

* Exposed for TESTING */ - void setup(HTableFactory factory, ExecutorService pool, Abortable abortable, Stoppable stop) { + void setup(HTableFactory factory, ExecutorService pool, Abortable abortable, Stoppable stop, + int cacheSize) { this.writerPool = MoreExecutors.listeningDecorator(pool); - this.factory = new CachingHTableFactory(factory); + this.factory = new CachingHTableFactory(factory, cacheSize); this.abortable = new CapturingAbortable(abortable); this.stopped = stop; } diff --git a/src/test/java/com/salesforce/hbase/index/write/TestCachingHTableFactory.java b/src/test/java/com/salesforce/hbase/index/write/TestCachingHTableFactory.java new file mode 100644 index 00000000..04cbbadd --- /dev/null +++ b/src/test/java/com/salesforce/hbase/index/write/TestCachingHTableFactory.java @@ -0,0 +1,68 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index.write; + +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.mockito.Mockito; + +import com.salesforce.hbase.index.table.CachingHTableFactory; +import com.salesforce.hbase.index.table.HTableFactory; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; + +public class TestCachingHTableFactory { + + @Test + public void testCacheCorrectlyExpiresTable() throws Exception { + // setup the mocks for the tables we will request + HTableFactory delegate = Mockito.mock(HTableFactory.class); + ImmutableBytesPtr t1 = new ImmutableBytesPtr(Bytes.toBytes("t1")); + ImmutableBytesPtr t2 = new ImmutableBytesPtr(Bytes.toBytes("t2")); + ImmutableBytesPtr t3 = new ImmutableBytesPtr(Bytes.toBytes("t3")); + HTableInterface table1 = Mockito.mock(HTableInterface.class); + HTableInterface table2 = Mockito.mock(HTableInterface.class); + HTableInterface table3 = Mockito.mock(HTableInterface.class); + Mockito.when(delegate.getTable(t1)).thenReturn(table1); + Mockito.when(delegate.getTable(t2)).thenReturn(table2); + Mockito.when(delegate.getTable(t3)).thenReturn(table3); + + // setup our factory with a cache size of 2 + CachingHTableFactory factory = new CachingHTableFactory(delegate, 2); + factory.getTable(t1); + factory.getTable(t2); + factory.getTable(t3); + // get the same table a second time, after it has gone out of cache + factory.getTable(t1); + + Mockito.verify(delegate, Mockito.times(2)).getTable(t1); + Mockito.verify(delegate, Mockito.times(1)).getTable(t2); + Mockito.verify(delegate, Mockito.times(1)).getTable(t3); + Mockito.verify(table1).close(); + } +} \ No newline at end of file diff --git a/src/test/java/com/salesforce/hbase/index/write/TestIndexWriter.java b/src/test/java/com/salesforce/hbase/index/write/TestIndexWriter.java index e523b6e0..f966e01b 100644 --- a/src/test/java/com/salesforce/hbase/index/write/TestIndexWriter.java +++ b/src/test/java/com/salesforce/hbase/index/write/TestIndexWriter.java @@ -126,7 +126,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable { // setup the writer and failure policy ParallelWriterIndexCommitter committer = new ParallelWriterIndexCommitter(); - committer.setup(factory, exec, abort, stop); + committer.setup(factory, exec, abort, stop, 2); KillServerOnFailurePolicy policy = new KillServerOnFailurePolicy(); policy.setup(stop, abort); IndexWriter writer = new IndexWriter(committer, policy); @@ -199,7 +199,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable { tables.put(new ImmutableBytesPtr(tableName2), table2); ParallelWriterIndexCommitter committer = new ParallelWriterIndexCommitter(); - committer.setup(factory, exec, abort, stop); + committer.setup(factory, exec, abort, stop, 2); KillServerOnFailurePolicy policy = new KillServerOnFailurePolicy(); policy.setup(stop, abort); IndexWriter writer = new IndexWriter(committer, policy); @@ -259,7 +259,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable { // setup the writer ParallelWriterIndexCommitter committer = new ParallelWriterIndexCommitter(); - committer.setup(factory, exec, abort, stop); + committer.setup(factory, exec, abort, stop, 2); KillServerOnFailurePolicy policy = new KillServerOnFailurePolicy(); policy.setup(stop, abort); final IndexWriter writer = new IndexWriter(committer, policy); diff --git a/src/test/java/com/salesforce/hbase/index/write/TestParalleWriterIndexCommitter.java b/src/test/java/com/salesforce/hbase/index/write/TestParalleWriterIndexCommitter.java new file mode 100644 index 00000000..ca3fed7c --- /dev/null +++ b/src/test/java/com/salesforce/hbase/index/write/TestParalleWriterIndexCommitter.java @@ -0,0 +1,127 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index.write; + +import static org.junit.Assert.assertTrue; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; +import com.salesforce.hbase.index.StubAbortable; +import com.salesforce.hbase.index.TableName; +import com.salesforce.hbase.index.table.HTableInterfaceReference; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; + +public class TestParalleWriterIndexCommitter { + + private static final Log LOG = LogFactory.getLog(TestParalleWriterIndexCommitter.class); + @Rule + public TableName test = new TableName(); + private final byte[] row = Bytes.toBytes("row"); + + @Test + public void testCorrectlyCleansUpResources() throws Exception{ + ExecutorService exec = Executors.newFixedThreadPool(1); + FakeTableFactory factory = new FakeTableFactory( + Collections. emptyMap()); + ParallelWriterIndexCommitter writer = new ParallelWriterIndexCommitter(); + Abortable mockAbort = Mockito.mock(Abortable.class); + Stoppable mockStop = Mockito.mock(Stoppable.class); + // create a simple writer + writer.setup(factory, exec, mockAbort, mockStop, 1); + // stop the writer + writer.stop(this.test.getTableNameString() + " finished"); + assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown); + assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown()); + Mockito.verifyZeroInteractions(mockAbort, mockStop); + } + + @SuppressWarnings("unchecked") + @Test + public void testSynchronouslyCompletesAllWrites() throws Exception { + LOG.info("Starting " + test.getTableNameString()); + LOG.info("Current thread is interrupted: " + Thread.interrupted()); + Abortable abort = new StubAbortable(); + Stoppable stop = Mockito.mock(Stoppable.class); + ExecutorService exec = Executors.newFixedThreadPool(1); + Map tables = + new HashMap(); + FakeTableFactory factory = new FakeTableFactory(tables); + + ImmutableBytesPtr tableName = new ImmutableBytesPtr(this.test.getTableName()); + Put m = new Put(row); + m.add(Bytes.toBytes("family"), Bytes.toBytes("qual"), null); + Multimap indexUpdates = + ArrayListMultimap. create(); + indexUpdates.put(new HTableInterfaceReference(tableName), m); + + HTableInterface table = Mockito.mock(HTableInterface.class); + final boolean[] completed = new boolean[] { false }; + Mockito.when(table.batch(Mockito.anyList())).thenAnswer(new Answer() { + + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + // just keep track that it was called + completed[0] = true; + return null; + } + }); + Mockito.when(table.getTableName()).thenReturn(test.getTableName()); + // add the table to the set of tables, so its returned to the writer + tables.put(tableName, table); + + // setup the writer and failure policy + ParallelWriterIndexCommitter writer = new ParallelWriterIndexCommitter(); + writer.setup(factory, exec, abort, stop, 1); + writer.write(indexUpdates); + assertTrue("Writer returned before the table batch completed! Likely a race condition tripped", + completed[0]); + writer.stop(this.test.getTableNameString() + " finished"); + assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown); + assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown()); + } +} \ No newline at end of file From 3f42a74b0579cb42e61f3d7da42ae58b8317fe71 Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Wed, 25 Sep 2013 14:49:52 -0700 Subject: [PATCH 074/102] Bug fix for TestParallelIndexWriter from merge --- .../salesforce/hbase/index/write/TestParalleIndexWriter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/salesforce/hbase/index/write/TestParalleIndexWriter.java b/src/test/java/com/salesforce/hbase/index/write/TestParalleIndexWriter.java index 3937e34d..40a603e4 100644 --- a/src/test/java/com/salesforce/hbase/index/write/TestParalleIndexWriter.java +++ b/src/test/java/com/salesforce/hbase/index/write/TestParalleIndexWriter.java @@ -72,7 +72,7 @@ public void testCorrectlyCleansUpResources() throws Exception{ Abortable mockAbort = Mockito.mock(Abortable.class); Stoppable mockStop = Mockito.mock(Stoppable.class); // create a simple writer - writer.setup(factory, exec, mockAbort, mockStop); + writer.setup(factory, exec, mockAbort, mockStop, 1); // stop the writer writer.stop(this.test.getTableNameString() + " finished"); assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown); @@ -116,7 +116,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable { // setup the writer and failure policy ParallelWriterIndexCommitter writer = new ParallelWriterIndexCommitter(); - writer.setup(factory, exec, abort, stop); + writer.setup(factory, exec, abort, stop, 1); writer.write(indexUpdates); assertTrue("Writer returned before the table batch completed! Likely a race condition tripped", completed[0]); From 47b470795071fab35703a1e20e05f7dadd8fdf3d Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Wed, 25 Sep 2013 18:44:09 -0700 Subject: [PATCH 075/102] Speeding up IndexMemStore, removing ExposedMemStore. Lots of time was spent creating a config in IndexMemStore that we didn't need - removing it. Also, removing the timetracker stuff - we don't care about whether or not to get a scanner for the store, given a timerange - we always scan the store. Additionally, removing ExposedMemStore since we are using our own impl now. --- .../index/covered/data/IndexMemStore.java | 19 +--- .../scanner/FilteredKeyValueScanner.java | 10 +- .../hbase/regionserver/ExposedMemStore.java | 92 ------------------- 3 files changed, 10 insertions(+), 111 deletions(-) delete mode 100644 src/main/java/org/apache/hadoop/hbase/regionserver/ExposedMemStore.java diff --git a/src/main/java/com/salesforce/hbase/index/covered/data/IndexMemStore.java b/src/main/java/com/salesforce/hbase/index/covered/data/IndexMemStore.java index 7a29c3fa..26ceb4cb 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/data/IndexMemStore.java +++ b/src/main/java/com/salesforce/hbase/index/covered/data/IndexMemStore.java @@ -40,7 +40,6 @@ import org.apache.hadoop.hbase.regionserver.KeyValueScanner; import org.apache.hadoop.hbase.regionserver.MemStore; import org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner; -import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; import org.apache.hadoop.hbase.util.Bytes; import com.salesforce.hbase.index.covered.KeyValueStore; @@ -79,7 +78,6 @@ public class IndexMemStore implements KeyValueStore { private static final Log LOG = LogFactory.getLog(IndexMemStore.class); private IndexKeyValueSkipListSet kvset; - private TimeRangeTracker timeRangeTracker = new TimeRangeTracker(); private Comparator comparator; /** @@ -115,22 +113,15 @@ public void add(KeyValue kv, boolean overwrite) { LOG.info("Inserting: " + toString(kv)); } // if overwriting, we will always update - boolean updated = true; if (!overwrite) { // null if there was no previous value, so we added the kv - updated = (kvset.putIfAbsent(kv) == null); + kvset.putIfAbsent(kv); } else { kvset.add(kv); } - // TODO do we even need to update this? I don't think we really even use it - // if we updated, we need to do some tracking work - if (updated) { - // update the max timestamp - this.timeRangeTracker.includeTimestamp(kv); - if (LOG.isDebugEnabled()) { - dump(); - } + if (LOG.isDebugEnabled()) { + dump(); } } @@ -306,8 +297,8 @@ public long getSequenceID() { @Override public boolean shouldUseScanner(Scan scan, SortedSet columns, long oldestUnexpiredTS) { - return (timeRangeTracker.includesTimeRange(scan.getTimeRange()) && (timeRangeTracker - .getMaximumTimestamp() >= oldestUnexpiredTS)); + throw new UnsupportedOperationException(this.getClass().getName() + + " doesn't support checking to see if it should use a scanner!"); } } } \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/scanner/FilteredKeyValueScanner.java b/src/main/java/com/salesforce/hbase/index/scanner/FilteredKeyValueScanner.java index af6d993b..c90319fd 100644 --- a/src/main/java/com/salesforce/hbase/index/scanner/FilteredKeyValueScanner.java +++ b/src/main/java/com/salesforce/hbase/index/scanner/FilteredKeyValueScanner.java @@ -7,16 +7,15 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.Filter.ReturnCode; -import org.apache.hadoop.hbase.regionserver.ExposedMemStore; import org.apache.hadoop.hbase.regionserver.KeyValueScanner; import com.salesforce.hbase.index.covered.KeyValueStore; /** * Combine a simplified version of the logic in the ScanQueryMatcher and the KeyValueScanner. We can - * get away with this here because we are only concerned with a single {@link ExposedMemStore} for - * the index; we don't need to worry about multiple column families or minimizing seeking through - * file - we just want to iterate the kvs quickly, in-memory. + * get away with this here because we are only concerned with a single MemStore for the index; we + * don't need to worry about multiple column families or minimizing seeking through file - we just + * want to iterate the kvs quickly, in-memory. */ public class FilteredKeyValueScanner implements KeyValueScanner { @@ -113,7 +112,8 @@ public long getSequenceID() { @Override public boolean shouldUseScanner(Scan scan, SortedSet columns, long oldestUnexpiredTS) { - return this.delegate.shouldUseScanner(scan, columns, oldestUnexpiredTS); + throw new UnsupportedOperationException(this.getClass().getName() + + " doesn't support checking to see if it should use a scanner!"); } diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/ExposedMemStore.java b/src/main/java/org/apache/hadoop/hbase/regionserver/ExposedMemStore.java deleted file mode 100644 index b97be293..00000000 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/ExposedMemStore.java +++ /dev/null @@ -1,92 +0,0 @@ -package org.apache.hadoop.hbase.regionserver; - -import java.rmi.UnexpectedException; -import java.util.List; -import java.util.SortedSet; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValue.KVComparator; - -/** - * A {@link MemStore} that exposes all the package-protected methods. - *

- * If used improperly, this class can completely hose the memstore - its only for expert usage. - *

- * @see MemStore - */ -public class ExposedMemStore extends MemStore { - - public ExposedMemStore(Configuration conf, KVComparator comparator) { - super(conf, comparator); - } - - @Override - public void dump() { - super.dump(); - } - - @Override - void snapshot() { - super.snapshot(); - } - - @Override - public KeyValueSkipListSet getSnapshot() { - return super.getSnapshot(); - } - - @Override - public void clearSnapshot(SortedSet ss) throws UnexpectedException { - super.clearSnapshot(ss); - } - - - @Override - public long add(KeyValue kv) { - return super.add(kv); - } - - @Override - public void rollback(KeyValue kv) { - super.rollback(kv); - } - - /** - * Use {@link #add(KeyValue)} instead! This method is not used in the usual HBase codepaths for - * adding deletes - */ - @Override - public long delete(KeyValue delete) { - return super.delete(delete); - } - - @Override - public KeyValue getNextRow(KeyValue kv) { - return getNextRow(kv); - } - - @Override - void getRowKeyAtOrBefore(GetClosestRowBeforeTracker state) { - super.getRowKeyAtOrBefore(state); - } - - @Override - public List getScanners() { - return super.getScanners(); - } - - @Override - public long heapSizeChange(KeyValue kv, boolean notpresent) { - return super.heapSizeChange(kv, notpresent); - } - - /** - * Disable the memstore MemSLAB in the given configuration. - * @param conf to update - */ - public static void disableMemSLAB(Configuration conf) { - conf.setBoolean(MemStore.USEMSLAB_KEY, false); - } - -} From 842f3dd42b99846201883f2871ba89714322c82e Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Wed, 25 Sep 2013 18:45:12 -0700 Subject: [PATCH 076/102] Switching ValueGetter to use ImmutableBytesPtr. There is a bunch of overhead associated with copying byte[]s. We now just return ImmutableBytePtrs and store ptrs, when at all possible. --- .../salesforce/hbase/index/ValueGetter.java | 3 +- .../hbase/index/covered/LocalTableState.java | 6 -- .../index/covered/data/LazyValueGetter.java | 13 ++-- .../hbase/index/util/IndexManagementUtil.java | 70 ++++++++++++++++--- .../phoenix/index/IndexMaintainer.java | 15 ++-- .../salesforce/phoenix/util/IndexUtil.java | 5 +- .../phoenix/index/IndexMaintainerTest.java | 5 +- 7 files changed, 82 insertions(+), 35 deletions(-) diff --git a/src/main/java/com/salesforce/hbase/index/ValueGetter.java b/src/main/java/com/salesforce/hbase/index/ValueGetter.java index f78ca385..0cf05886 100644 --- a/src/main/java/com/salesforce/hbase/index/ValueGetter.java +++ b/src/main/java/com/salesforce/hbase/index/ValueGetter.java @@ -30,6 +30,7 @@ import java.io.IOException; import com.salesforce.hbase.index.covered.update.ColumnReference; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; public interface ValueGetter { @@ -41,5 +42,5 @@ public interface ValueGetter { * present. * @throws IOException if there is an error accessing the underlying data storage */ - public byte[] getLatestValue(ColumnReference ref) throws IOException; + public ImmutableBytesPtr getLatestValue(ColumnReference ref) throws IOException; } \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java b/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java index d7598664..bdee8e53 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java +++ b/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java @@ -36,12 +36,10 @@ import java.util.Map; import java.util.Set; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -import org.apache.hadoop.hbase.regionserver.ExposedMemStore; import org.apache.hadoop.hbase.regionserver.KeyValueScanner; import org.apache.hadoop.hbase.util.Pair; @@ -80,10 +78,6 @@ public LocalTableState(RegionCoprocessorEnvironment environment, LocalHBaseState this.env = environment; this.table = table; this.update = update; - // ensure that the memstore just uses the KV references, rather than copying them into the - // memstore - Configuration conf = new Configuration(environment.getConfiguration()); - ExposedMemStore.disableMemSLAB(conf); this.memstore = new IndexMemStore(IndexMemStore.COMPARATOR); this.scannerBuilder = new ScannerBuilder(memstore, update); this.columnSet = new CoveredColumns(); diff --git a/src/main/java/com/salesforce/hbase/index/covered/data/LazyValueGetter.java b/src/main/java/com/salesforce/hbase/index/covered/data/LazyValueGetter.java index 76c98c25..2f380370 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/data/LazyValueGetter.java +++ b/src/main/java/com/salesforce/hbase/index/covered/data/LazyValueGetter.java @@ -37,6 +37,7 @@ import com.salesforce.hbase.index.ValueGetter; import com.salesforce.hbase.index.covered.update.ColumnReference; import com.salesforce.hbase.index.scanner.Scanner; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; /** * {@link ValueGetter} that uses lazy initialization to get the value for the given @@ -45,7 +46,7 @@ public class LazyValueGetter implements ValueGetter { private Scanner scan; - private volatile Map values; + private volatile Map values; private byte[] row; /** @@ -59,16 +60,16 @@ public LazyValueGetter(Scanner scan, byte[] currentRow) { } @Override - public byte[] getLatestValue(ColumnReference ref) throws IOException { + public ImmutableBytesPtr getLatestValue(ColumnReference ref) throws IOException { // ensure we have a backing map if (values == null) { synchronized (this) { - values = Collections.synchronizedMap(new HashMap()); + values = Collections.synchronizedMap(new HashMap()); } } // check the value in the map - byte[] value = values.get(ref); + ImmutableBytesPtr value = values.get(ref); if (value == null) { value = get(ref); values.put(ref, value); @@ -81,7 +82,7 @@ public byte[] getLatestValue(ColumnReference ref) throws IOException { * @param ref * @return the first value on the scanner for the given column */ - private byte[] get(ColumnReference ref) throws IOException { + private ImmutableBytesPtr get(ColumnReference ref) throws IOException { KeyValue first = ref.getFirstKeyValueForRow(row); if (!scan.seek(first)) { return null; @@ -89,7 +90,7 @@ private byte[] get(ColumnReference ref) throws IOException { // there is a next value - we only care about the current value, so we can just snag that KeyValue next = scan.next(); if (ref.matches(next)) { - return next.getValue(); + return new ImmutableBytesPtr(next.getBuffer(), next.getValueOffset(), next.getValueLength()); } return null; } diff --git a/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java b/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java index 9aa284dd..4205d50c 100644 --- a/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java +++ b/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java @@ -49,19 +49,67 @@ private IndexManagementUtil() { // private ctor for util classes } - public static ValueGetter createGetterFromKeyValues (Collection pendingUpdates) { - final Map valueMap = Maps.newHashMapWithExpectedSize(pendingUpdates.size()); - for (KeyValue kv : pendingUpdates) { - valueMap.put(new ColumnReference(kv.getFamily(), kv.getQualifier()), kv.getValue()); + public static ValueGetter createGetterFromKeyValues(Collection pendingUpdates) { + final Map valueMap = + Maps.newHashMapWithExpectedSize(pendingUpdates.size()); + for (KeyValue kv : pendingUpdates) { + // create new pointers to each part of the kv + ImmutableBytesPtr family = + new ImmutableBytesPtr(kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength()); + ImmutableBytesPtr qual = + new ImmutableBytesPtr(kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength()); + ImmutableBytesPtr value = + new ImmutableBytesPtr(kv.getBuffer(), kv.getValueOffset(), kv.getValueLength()); + valueMap.put(new ReferencingColumn(family, qual), value); + } + return new ValueGetter() { + @Override + public ImmutableBytesPtr getLatestValue(ColumnReference ref) throws IOException { + return valueMap.get(ReferencingColumn.wrap(ref)); } - return new ValueGetter() { - @Override - public byte[] getLatestValue(ColumnReference ref) throws IOException { - return valueMap.get(ref); - } - }; - + }; } + + private static class ReferencingColumn { + ImmutableBytesPtr family; + ImmutableBytesPtr qual; + + static ReferencingColumn wrap(ColumnReference ref) { + ImmutableBytesPtr family = new ImmutableBytesPtr(ref.getFamily()); + ImmutableBytesPtr qual = new ImmutableBytesPtr(ref.getQualifier()); + return new ReferencingColumn(family, qual); + } + + public ReferencingColumn(ImmutableBytesPtr family, ImmutableBytesPtr qual) { + this.family = family; + this.qual = qual; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((family == null) ? 0 : family.hashCode()); + result = prime * result + ((qual == null) ? 0 : qual.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + ReferencingColumn other = (ReferencingColumn) obj; + if (family == null) { + if (other.family != null) return false; + } else if (!family.equals(other.family)) return false; + if (qual == null) { + if (other.qual != null) return false; + } else if (!qual.equals(other.qual)) return false; + return true; + } + } + public static ValueGetter createGetterFromScanner(Scanner scanner, byte[] currentRow) { return new LazyValueGetter(scanner, currentRow); } diff --git a/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java b/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java index 291935f2..0ce06f63 100644 --- a/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java +++ b/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java @@ -23,6 +23,7 @@ import com.google.common.collect.Lists; import com.salesforce.hbase.index.ValueGetter; import com.salesforce.hbase.index.covered.update.ColumnReference; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.hbase.index.util.IndexManagementUtil; import com.salesforce.phoenix.query.QueryConstants; import com.salesforce.phoenix.schema.ColumnModifier; @@ -238,11 +239,11 @@ public byte[] buildRowKey(ValueGetter valueGetter, ImmutableBytesWritable rowKey ColumnModifier dataColumnModifier = null; if (dataPkPosition[i] == -1) { dataColumnType = indexedColumnTypes.get(j); - byte[] value = valueGetter.getLatestValue(indexedColumns.get(j)); + ImmutableBytesPtr value = valueGetter.getLatestValue(indexedColumns.get(j)); if (value == null) { ptr.set(ByteUtil.EMPTY_BYTE_ARRAY); } else { - ptr.set(value); + ptr.set(value.copyBytesIfNecessary()); } j++; } else { @@ -301,8 +302,8 @@ public Put buildUpdateMutation(ValueGetter valueGetter, ImmutableBytesWritable d for (int i = 0; i < this.getCoverededColumns().size(); i++) { ColumnReference ref = this.getCoverededColumns().get(i); byte[] cq = this.indexQualifiers.get(i); - byte[] value = valueGetter.getLatestValue(ref); - put.add(ref.getFamily(), cq, ts, value); + ImmutableBytesPtr value = valueGetter.getLatestValue(ref); + put.add(ref.getFamily(), cq, ts, value == null ? null : value.copyBytesIfNecessary()); } // Add the empty key value put.add(this.getEmptyKeyValueFamily(), QueryConstants.EMPTY_COLUMN_BYTES, ts, ByteUtil.EMPTY_BYTE_ARRAY); @@ -320,10 +321,10 @@ public Delete buildDeleteMutation(ValueGetter valueGetter, ImmutableBytesWritabl private boolean indexedColumnsChanged(ValueGetter oldState, ValueGetter newState) throws IOException { for (int i = 0; i < indexedColumns.size(); i++) { ColumnReference ref = indexedColumns.get(i); - byte[] newValue = newState.getLatestValue(ref); + ImmutableBytesPtr newValue = newState.getLatestValue(ref); if (newValue != null) { // Indexed column was changed - byte[] oldValue = oldState.getLatestValue(ref); - if (oldValue == null || !Arrays.equals(newValue, oldValue)) { + ImmutableBytesPtr oldValue = oldState.getLatestValue(ref); + if (oldValue == null || !oldValue.equals(newValue)){ return true; } } diff --git a/src/main/java/com/salesforce/phoenix/util/IndexUtil.java b/src/main/java/com/salesforce/phoenix/util/IndexUtil.java index 088ecf41..677af61e 100644 --- a/src/main/java/com/salesforce/phoenix/util/IndexUtil.java +++ b/src/main/java/com/salesforce/phoenix/util/IndexUtil.java @@ -41,6 +41,7 @@ import com.google.common.collect.Lists; import com.salesforce.hbase.index.ValueGetter; import com.salesforce.hbase.index.covered.update.ColumnReference; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.exception.SQLExceptionCode; import com.salesforce.phoenix.exception.SQLExceptionInfo; import com.salesforce.phoenix.index.IndexMaintainer; @@ -144,7 +145,7 @@ public static List generateIndexData(PTable table, PTable index, List< ValueGetter valueGetter = new ValueGetter() { @Override - public byte[] getLatestValue(ColumnReference ref) { + public ImmutableBytesPtr getLatestValue(ColumnReference ref) { Map> familyMap = dataMutation.getFamilyMap(); byte[] family = ref.getFamily(); List kvs = familyMap.get(family); @@ -155,7 +156,7 @@ public byte[] getLatestValue(ColumnReference ref) { for (KeyValue kv : kvs) { if (Bytes.compareTo(kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(), family, 0, family.length) == 0 && Bytes.compareTo(kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength(), qualifier, 0, qualifier.length) == 0) { - return kv.getValue(); + return new ImmutableBytesPtr(kv.getBuffer(), kv.getValueOffset(), kv.getValueLength()); } } return null; diff --git a/src/test/java/com/salesforce/phoenix/index/IndexMaintainerTest.java b/src/test/java/com/salesforce/phoenix/index/IndexMaintainerTest.java index cf7e90d8..37446676 100644 --- a/src/test/java/com/salesforce/phoenix/index/IndexMaintainerTest.java +++ b/src/test/java/com/salesforce/phoenix/index/IndexMaintainerTest.java @@ -22,6 +22,7 @@ import com.google.common.collect.Maps; import com.salesforce.hbase.index.ValueGetter; import com.salesforce.hbase.index.covered.update.ColumnReference; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.phoenix.end2end.index.IndexTestUtil; import com.salesforce.phoenix.jdbc.PhoenixConnection; import com.salesforce.phoenix.query.BaseConnectionlessQueryTest; @@ -56,8 +57,8 @@ private static ValueGetter newValueGetter(final Map val return new ValueGetter() { @Override - public byte[] getLatestValue(ColumnReference ref) { - return valueMap.get(ref); + public ImmutableBytesPtr getLatestValue(ColumnReference ref) { + return new ImmutableBytesPtr(valueMap.get(ref)); } }; From 02fa010e44ecd9b9f0ee8d056b97fad14b62f041 Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Wed, 25 Sep 2013 18:52:13 -0700 Subject: [PATCH 077/102] Adding covered columns back to covered set when found. We had the infrastructure to track the columns that have already been loaded, but never actually updated the set of columns that were loaded. This meant that we reloaded the same columns over and over, even though they had already been loaded --- .../com/salesforce/hbase/index/covered/LocalTableState.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java b/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java index bdee8e53..3794f81b 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java +++ b/src/main/java/com/salesforce/hbase/index/covered/LocalTableState.java @@ -177,6 +177,11 @@ private synchronized void ensureLocalStateInitialized( // add the current state of the row this.addUpdate(this.table.getCurrentRowState(update, toCover).list(), false); + + // add the covered columns to the set + for (ColumnReference ref : toCover) { + this.columnSet.addColumn(ref); + } } @Override From 079dcb1eecf2621e87fd82e5008b4c0ec8d16e96 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Wed, 25 Sep 2013 23:05:22 -0700 Subject: [PATCH 078/102] Comment out test to get perf numbers --- .../com/salesforce/hbase/index/covered/TestLocalTableState.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java b/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java index 7016bc00..981a761b 100644 --- a/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java +++ b/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java @@ -108,7 +108,7 @@ public Boolean answer(InvocationOnMock invocation) throws Throwable { * Test that we correctly rollback the state of a keyvalue if its a {@link PendingKeyValue}. * @throws Exception */ - @Test + // @Test temporary to get perf numbers @SuppressWarnings("unchecked") public void testCorrectRollback() throws Exception { Put m = new Put(row); From 34996f46eaffcd032a118641a6702adf758ac545 Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Thu, 26 Sep 2013 09:16:28 -0700 Subject: [PATCH 079/102] Fixing bug in TestLocalTableState Forgot to cleanup the previous test from when we were using PendingKeyValue to manage state in the memstore. Now, correctly tests rolling back plus some extra test around not loading columns. --- .../index/covered/data/IndexMemStore.java | 22 ++++++-- .../index/covered/TestCoveredColumns.java | 55 ++++++++++++++++++ .../index/covered/TestLocalTableState.java | 56 +++++++++++++++++-- 3 files changed, 123 insertions(+), 10 deletions(-) create mode 100644 src/test/java/com/salesforce/hbase/index/covered/TestCoveredColumns.java diff --git a/src/main/java/com/salesforce/hbase/index/covered/data/IndexMemStore.java b/src/main/java/com/salesforce/hbase/index/covered/data/IndexMemStore.java index 26ceb4cb..9e33be66 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/data/IndexMemStore.java +++ b/src/main/java/com/salesforce/hbase/index/covered/data/IndexMemStore.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.util.Bytes; import com.salesforce.hbase.index.covered.KeyValueStore; +import com.salesforce.hbase.index.covered.LocalTableState; /** * Like the HBase {@link MemStore}, but without all that extra work around maintaining snapshots and @@ -59,20 +60,27 @@ * * *

  • ignoring memstore timestamps in favor of deciding when we want to overwrite keys based on how - * we obtain them
    • + * we obtain them + *
  • ignoring time range updates (so + * {@link KeyValueScanner#shouldUseScanner(Scan, SortedSet, long)} isn't supported from + * {@link #getScanner()}).
    • * *

    * We can ignore the memstore timestamps because we know that anything we get from the local region * is going to be MVCC visible - so it should just go in. However, we also want overwrite any * existing state with our pending write that we are indexing, so that needs to clobber the KVs we * get from the HRegion. This got really messy with a regular memstore as each KV from the MemStore - * frequently has a higher MemStoreTS, but we can't just up the pending KVs' MemStoreTs b/c a - * memstore relies on the MVCC readpoint, which generally is < Long.MAX_VALUE. + * frequently has a higher MemStoreTS, but we can't just up the pending KVs' MemStoreTs because a + * memstore relies on the MVCC readpoint, which generally is less than {@link Long#MAX_VALUE}. *

    * By realizing that we don't need the snapshot or space requirements, we can go much faster than * the previous implementation. Further, by being smart about how we manage the KVs, we can drop the * extra object creation we were doing to wrap the pending KVs (which we did previously to ensure - * they sorted before the ones we got from the HRegion). + * they sorted before the ones we got from the HRegion). We overwrite {@link KeyValue}s when we add + * them from external sources {@link #add(KeyValue, boolean)}, but then don't overwrite existing + * keyvalues when read them from the underlying table (because pending keyvalues should always + * overwrite current ones) - this logic is all contained in LocalTableState. + * @see LocalTableState */ public class IndexMemStore implements KeyValueStore { @@ -139,8 +147,14 @@ private String toString(KeyValue kv) { @Override public void rollback(KeyValue kv) { + if (LOG.isDebugEnabled()) { + LOG.debug("Rolling back: " + toString(kv)); + } // If the key is in the store, delete it this.kvset.remove(kv); + if (LOG.isDebugEnabled()) { + dump(); + } } @Override diff --git a/src/test/java/com/salesforce/hbase/index/covered/TestCoveredColumns.java b/src/test/java/com/salesforce/hbase/index/covered/TestCoveredColumns.java new file mode 100644 index 00000000..e1cc9746 --- /dev/null +++ b/src/test/java/com/salesforce/hbase/index/covered/TestCoveredColumns.java @@ -0,0 +1,55 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index.covered; + +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; + +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; + +import com.salesforce.hbase.index.covered.update.ColumnReference; + +public class TestCoveredColumns { + + private static final byte[] fam = Bytes.toBytes("fam"); + private static final byte[] qual = Bytes.toBytes("qual"); + + @Test + public void testCovering() { + ColumnReference ref = new ColumnReference(fam, qual); + CoveredColumns columns = new CoveredColumns(); + assertEquals("Should have only found a single column to cover", 1, columns + .findNonCoveredColumns(Arrays.asList(ref)).size()); + + columns.addColumn(ref); + assertEquals("Shouldn't have any columns to cover", 0, + columns.findNonCoveredColumns(Arrays.asList(ref)).size()); + } +} \ No newline at end of file diff --git a/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java b/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java index 7016bc00..96d122aa 100644 --- a/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java +++ b/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java @@ -105,7 +105,7 @@ public Boolean answer(InvocationOnMock invocation) throws Throwable { } /** - * Test that we correctly rollback the state of a keyvalue if its a {@link PendingKeyValue}. + * Test that we correctly rollback the state of keyvalue * @throws Exception */ @Test @@ -114,10 +114,7 @@ public void testCorrectRollback() throws Exception { Put m = new Put(row); m.add(fam, qual, ts, val); // setup mocks - Configuration conf = new Configuration(false); RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class); - Mockito.when(env.getConfiguration()).thenReturn(conf); - Mockito.when(env.getConfiguration()).thenReturn(conf); HRegion region = Mockito.mock(HRegion.class); Mockito.when(env.getRegion()).thenReturn(region); @@ -154,6 +151,53 @@ public Boolean answer(InvocationOnMock invocation) throws Throwable { table.rollback(Arrays.asList(kv)); p = table.getIndexedColumnsTableState(Arrays.asList(col)); s = p.getFirst(); - assertEquals("Didn't correctly rollback the row - still found it!", storedKv, s.next()); + assertEquals("Didn't correctly rollback the row - still found it!", null, s.next()); + Mockito.verify(env, Mockito.times(1)).getRegion(); + Mockito.verify(region, Mockito.times(1)).getScanner(Mockito.any(Scan.class)); + } + + @SuppressWarnings("unchecked") + @Test + public void testOnlyLoadsRequestedColumns() throws Exception { + // setup mocks + RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class); + + HRegion region = Mockito.mock(HRegion.class); + Mockito.when(env.getRegion()).thenReturn(region); + RegionScanner scanner = Mockito.mock(RegionScanner.class); + Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenReturn(scanner); + final KeyValue storedKv = + new KeyValue(row, fam, qual, ts, Type.Put, Bytes.toBytes("stored-value")); + storedKv.setMemstoreTS(2); + Mockito.when(scanner.next(Mockito.any(List.class))).thenAnswer(new Answer() { + @Override + public Boolean answer(InvocationOnMock invocation) throws Throwable { + List list = (List) invocation.getArguments()[0]; + + list.add(storedKv); + return false; + } + }); + LocalHBaseState state = new LocalTable(env); + Put pendingUpdate = new Put(row); + pendingUpdate.add(fam, qual, ts, val); + LocalTableState table = new LocalTableState(env, state, pendingUpdate); + + // do the lookup for the given column + ColumnReference col = new ColumnReference(fam, qual); + table.setCurrentTimestamp(ts); + // check that the value is there + Pair p = table.getIndexedColumnsTableState(Arrays.asList(col)); + Scanner s = p.getFirst(); + // make sure it read the table the one time + assertEquals("Didn't get the stored keyvalue!", storedKv, s.next()); + + // on the second lookup it shouldn't access the underlying table again - the cached columns + // should know they are done + p = table.getIndexedColumnsTableState(Arrays.asList(col)); + s = p.getFirst(); + assertEquals("Lost already loaded update!", storedKv, s.next()); + Mockito.verify(env, Mockito.times(1)).getRegion(); + Mockito.verify(region, Mockito.times(1)).getScanner(Mockito.any(Scan.class)); } -} +} \ No newline at end of file From 14c8dece1d4a5838eeb58a978db506e58ce36973 Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Thu, 26 Sep 2013 09:57:51 -0700 Subject: [PATCH 080/102] Adding TestLocalTableState back in, since fixed --- .../com/salesforce/hbase/index/covered/TestLocalTableState.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java b/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java index 7503264d..80eb8f59 100644 --- a/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java +++ b/src/test/java/com/salesforce/hbase/index/covered/TestLocalTableState.java @@ -108,7 +108,7 @@ public Boolean answer(InvocationOnMock invocation) throws Throwable { * Test that we correctly rollback the state of keyvalue * @throws Exception */ - // @Test temporary to get perf numbers + @Test @SuppressWarnings("unchecked") public void testCorrectRollback() throws Exception { Put m = new Put(row); From 60a80ee9f17b8c3619f1885e480cfc63d4c314b8 Mon Sep 17 00:00:00 2001 From: maryannxue Date: Thu, 26 Sep 2013 14:48:57 -0400 Subject: [PATCH 081/102] bug fix --- .../java/com/salesforce/phoenix/compile/StatementContext.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/salesforce/phoenix/compile/StatementContext.java b/src/main/java/com/salesforce/phoenix/compile/StatementContext.java index 736fdd68..49c9c2bd 100644 --- a/src/main/java/com/salesforce/phoenix/compile/StatementContext.java +++ b/src/main/java/com/salesforce/phoenix/compile/StatementContext.java @@ -91,7 +91,9 @@ public StatementContext(BindableStatement statement, PhoenixConnection connectio this.tempPtr = new ImmutableBytesWritable(); this.disambiguateWithTable = disambiguateWithTable; this.hashClient = hashClient; - this.currentTable = resolver.getTables().get(0); + if (!resolver.getTables().isEmpty()) { + this.currentTable = resolver.getTables().get(0); + } } public String getDateFormat() { From 96f619ef10f3299e671dd3f5dd53f7635895c77f Mon Sep 17 00:00:00 2001 From: maryannxue Date: Thu, 26 Sep 2013 15:09:11 -0400 Subject: [PATCH 082/102] bug fix --- .../java/com/salesforce/phoenix/compile/WhereCompiler.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/salesforce/phoenix/compile/WhereCompiler.java b/src/main/java/com/salesforce/phoenix/compile/WhereCompiler.java index 3899affd..57888a01 100644 --- a/src/main/java/com/salesforce/phoenix/compile/WhereCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/WhereCompiler.java @@ -146,6 +146,9 @@ public void increment(KeyValueColumnExpression column) { } public void increment(RowKeyColumnExpression column) { + if (column.getCFPrefix() == null) + return; + switch (count) { case NONE: count = Count.MULTIPLE; From 67c2646677c44cb3d2f51d4e8e6ad633653c5eb0 Mon Sep 17 00:00:00 2001 From: maryannxue Date: Thu, 26 Sep 2013 15:16:25 -0400 Subject: [PATCH 083/102] bug fix --- .../java/com/salesforce/phoenix/compile/StatementContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/salesforce/phoenix/compile/StatementContext.java b/src/main/java/com/salesforce/phoenix/compile/StatementContext.java index 49c9c2bd..3b9f5f3c 100644 --- a/src/main/java/com/salesforce/phoenix/compile/StatementContext.java +++ b/src/main/java/com/salesforce/phoenix/compile/StatementContext.java @@ -91,7 +91,7 @@ public StatementContext(BindableStatement statement, PhoenixConnection connectio this.tempPtr = new ImmutableBytesWritable(); this.disambiguateWithTable = disambiguateWithTable; this.hashClient = hashClient; - if (!resolver.getTables().isEmpty()) { + if (resolver != null && !resolver.getTables().isEmpty()) { this.currentTable = resolver.getTables().get(0); } } From f1636759b21db68da95733995653e2524fda5e96 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Thu, 26 Sep 2013 15:04:37 -0700 Subject: [PATCH 084/102] Change index column and covered column to set --- .../phoenix/index/IndexMaintainer.java | 116 +++++++++--------- .../phoenix/index/PhoenixIndexBuilder.java | 3 +- 2 files changed, 61 insertions(+), 58 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java b/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java index 0ce06f63..287a4284 100644 --- a/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java +++ b/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java @@ -9,7 +9,10 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; @@ -21,10 +24,11 @@ import org.apache.hadoop.io.WritableUtils; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.salesforce.hbase.index.ValueGetter; import com.salesforce.hbase.index.covered.update.ColumnReference; import com.salesforce.hbase.index.util.ImmutableBytesPtr; -import com.salesforce.hbase.index.util.IndexManagementUtil; import com.salesforce.phoenix.query.QueryConstants; import com.salesforce.phoenix.schema.ColumnModifier; import com.salesforce.phoenix.schema.PColumn; @@ -162,9 +166,9 @@ public static List deserialize(byte[] buf, int offset, int leng return maintainers; } - private List indexedColumns; - private List coveredColumns; - private List allColumns; + private Set indexedColumns; + private Set coveredColumns; + private Set allColumns; private List indexedColumnTypes; private List indexedColumnByteSizes; private RowKeyMetaData rowKeyMetaData; @@ -175,6 +179,7 @@ public static List deserialize(byte[] buf, int offset, int leng private final boolean isDataTableSalted; private final RowKeySchema dataRowKeySchema; + private byte[] emptyKeyValueCF; private List indexQualifiers; private int estimatedIndexRowKeyBytes; private int[][] dataRowKeyLocator; @@ -192,11 +197,11 @@ private IndexMaintainer(RowKeySchema dataRowKeySchema, boolean isDataTableSalted int nDataPKColumns = dataRowKeySchema.getFieldCount() - (isDataTableSalted ? 1 : 0); this.dataRowKeyLocator = new int[2][nIndexPKColumns]; this.indexTableName = indexTableName; - this.indexedColumns = Lists.newArrayListWithExpectedSize(nIndexPKColumns-nDataPKColumns); + this.indexedColumns = Sets.newLinkedHashSetWithExpectedSize(nIndexPKColumns-nDataPKColumns); this.indexedColumnTypes = Lists.newArrayListWithExpectedSize(nIndexPKColumns-nDataPKColumns); this.indexedColumnByteSizes = Lists.newArrayListWithExpectedSize(nIndexPKColumns-nDataPKColumns); - this.coveredColumns = Lists.newArrayListWithExpectedSize(nIndexColumns-nIndexPKColumns); - this.allColumns = Lists.newArrayListWithExpectedSize(nDataPKColumns + nIndexColumns); + this.coveredColumns = Sets.newLinkedHashSetWithExpectedSize(nIndexColumns-nIndexPKColumns); + this.allColumns = Sets.newLinkedHashSetWithExpectedSize(nDataPKColumns + nIndexColumns); this.allColumns.addAll(indexedColumns); this.allColumns.addAll(coveredColumns); this.rowKeyMetaData = newRowKeyMetaData(nIndexPKColumns); @@ -232,6 +237,7 @@ public byte[] buildRowKey(ValueGetter valueGetter, ImmutableBytesWritable rowKey } BitSet descIndexColumnBitSet = rowKeyMetaData.getDescIndexColumnBitSet(); int j = 0; + Iterator iterator = indexedColumns.iterator(); for (int i = 0; i < nIndexedColumns; i++) { PDataType dataColumnType; boolean isNullable = true; @@ -239,7 +245,7 @@ public byte[] buildRowKey(ValueGetter valueGetter, ImmutableBytesWritable rowKey ColumnModifier dataColumnModifier = null; if (dataPkPosition[i] == -1) { dataColumnType = indexedColumnTypes.get(j); - ImmutableBytesPtr value = valueGetter.getLatestValue(indexedColumns.get(j)); + ImmutableBytesPtr value = valueGetter.getLatestValue(iterator.next()); if (value == null) { ptr.set(ByteUtil.EMPTY_BYTE_ARRAY); } else { @@ -299,9 +305,9 @@ public byte[] buildRowKey(ValueGetter valueGetter, ImmutableBytesWritable rowKey public Put buildUpdateMutation(ValueGetter valueGetter, ImmutableBytesWritable dataRowKeyPtr, long ts) throws IOException { byte[] indexRowKey = this.buildRowKey(valueGetter, dataRowKeyPtr); Put put = new Put(indexRowKey); - for (int i = 0; i < this.getCoverededColumns().size(); i++) { - ColumnReference ref = this.getCoverededColumns().get(i); - byte[] cq = this.indexQualifiers.get(i); + int i = 0; + for (ColumnReference ref : this.getCoverededColumns()) { + byte[] cq = this.indexQualifiers.get(i++); ImmutableBytesPtr value = valueGetter.getLatestValue(ref); put.add(ref.getFamily(), cq, ts, value == null ? null : value.copyBytesIfNecessary()); } @@ -318,13 +324,21 @@ public Delete buildDeleteMutation(ValueGetter valueGetter, ImmutableBytesWritabl return buildDeleteMutation(valueGetter, dataRowKeyPtr, pendingUpdates, HConstants.LATEST_TIMESTAMP); } - private boolean indexedColumnsChanged(ValueGetter oldState, ValueGetter newState) throws IOException { - for (int i = 0; i < indexedColumns.size(); i++) { - ColumnReference ref = indexedColumns.get(i); - ImmutableBytesPtr newValue = newState.getLatestValue(ref); - if (newValue != null) { // Indexed column was changed + private boolean indexedColumnsChanged(ValueGetter oldState, Collection pendingUpdates) throws IOException { + if (pendingUpdates.isEmpty()) { + return false; + } + Map newState = Maps.newHashMapWithExpectedSize(pendingUpdates.size()); + for (KeyValue kv : pendingUpdates) { + newState.put(new ColumnReference(kv.getFamily(), kv.getQualifier()), kv); + } + for (ColumnReference ref : indexedColumns) { + KeyValue newValue = newState.get(ref); + if (newValue != null) { // Indexed column was potentially changed ImmutableBytesPtr oldValue = oldState.getLatestValue(ref); - if (oldValue == null || !oldValue.equals(newValue)){ + if (oldValue == null || + Bytes.compareTo(oldValue.get(), oldValue.getOffset(), oldValue.getLength(), + newValue.getBuffer(), newValue.getValueOffset(), newValue.getValueLength()) != 0){ return true; } } @@ -336,47 +350,39 @@ private boolean indexedColumnsChanged(ValueGetter oldState, ValueGetter newState public Delete buildDeleteMutation(ValueGetter oldState, ImmutableBytesWritable dataRowKeyPtr, Collection pendingUpdates, long ts) throws IOException { byte[] indexRowKey = this.buildRowKey(oldState, dataRowKeyPtr); // Delete the entire row if any of the indexed columns changed - if (pendingUpdates.isEmpty() || indexedColumnsChanged(oldState, IndexManagementUtil.createGetterFromKeyValues(pendingUpdates))) { // Deleting the entire row + if (indexedColumnsChanged(oldState, pendingUpdates)) { // Deleting the entire row Delete delete = new Delete(indexRowKey, ts, null); return delete; } + Delete delete = null; // Delete columns for missing key values - Delete delete = new Delete(indexRowKey); - boolean hasColumns = false; for (KeyValue kv : pendingUpdates) { - if (kv.getType() == KeyValue.Type.Put.getCode()) { - continue; - } - hasColumns = true; - byte[] family = kv.getFamily(); - byte[] qual = kv.getQualifier(); - byte[] indexQual = IndexUtil.getIndexColumnName(family, qual); - for (ColumnReference col : this.getCoverededColumns()) { - if (col.matches(kv)) { - delete.deleteColumns(family, indexQual, ts); + if (kv.getType() != KeyValue.Type.Put.getCode()) { + ColumnReference ref = new ColumnReference(kv.getFamily(), kv.getQualifier()); + if (coveredColumns.contains(ref)) { + if (delete == null) { + delete = new Delete(indexRowKey); + } + delete.deleteColumns(ref.getFamily(), IndexUtil.getIndexColumnName(ref.getFamily(), ref.getQualifier()), ts); } - } - } - // only return a delete if we need to cleanup some columns - if (hasColumns) { - return delete; + } } - return null; + return delete; } public byte[] getIndexTableName() { return indexTableName; } - public List getCoverededColumns() { + public Set getCoverededColumns() { return coveredColumns; } - public List getIndexedColumns() { + public Set getIndexedColumns() { return indexedColumns; } - public List getAllColumns() { + public Set getAllColumns() { return allColumns; } @@ -384,10 +390,7 @@ private byte[] getEmptyKeyValueFamily() { // Since the metadata of an index table will never change, // we can infer this based on the family of the first covered column // If if there are no covered columns, we know it's our default name - if (coveredColumns.isEmpty()) { - return QueryConstants.EMPTY_COLUMN_BYTES; - } - return coveredColumns.get(0).getFamily(); + return emptyKeyValueCF; } private RowKeyMetaData getRowKeyMetaData() { @@ -406,7 +409,7 @@ private List getIndexedColumnTypes() { public void readFields(DataInput input) throws IOException { nIndexSaltBuckets = WritableUtils.readVInt(input); int nIndexedColumns = WritableUtils.readVInt(input); - indexedColumns = Lists.newArrayListWithExpectedSize(nIndexedColumns); + indexedColumns = Sets.newLinkedHashSetWithExpectedSize(nIndexedColumns); for (int i = 0; i < nIndexedColumns; i++) { byte[] cf = Bytes.readByteArray(input); byte[] cq = Bytes.readByteArray(input); @@ -423,7 +426,7 @@ public void readFields(DataInput input) throws IOException { indexedColumnByteSizes.add(byteSize == 0 ? null : Integer.valueOf(byteSize)); } int nCoveredColumns = WritableUtils.readVInt(input); - coveredColumns = Lists.newArrayListWithExpectedSize(nCoveredColumns); + coveredColumns = Sets.newLinkedHashSetWithExpectedSize(nCoveredColumns); for (int i = 0; i < nCoveredColumns; i++) { byte[] cf = Bytes.readByteArray(input); byte[] cq = Bytes.readByteArray(input); @@ -448,14 +451,19 @@ private int estimateIndexRowKeyByteSize() { * Init calculated state reading/creating */ private void initCachedState() { + if (coveredColumns.isEmpty()) { + emptyKeyValueCF = QueryConstants.EMPTY_COLUMN_BYTES; + } else { + emptyKeyValueCF = coveredColumns.iterator().next().getFamily(); + } + indexQualifiers = Lists.newArrayListWithExpectedSize(this.coveredColumns.size()); - for (int i = 0; i < coveredColumns.size(); i++) { - ColumnReference ref = coveredColumns.get(i); + for (ColumnReference ref : coveredColumns) { indexQualifiers.add(IndexUtil.getIndexColumnName(ref.getFamily(), ref.getQualifier())); } estimatedIndexRowKeyBytes = estimateIndexRowKeyByteSize(); - this.allColumns = Lists.newArrayListWithExpectedSize(indexedColumns.size() + coveredColumns.size()); + this.allColumns = Sets.newLinkedHashSetWithExpectedSize(indexedColumns.size() + coveredColumns.size()); allColumns.addAll(indexedColumns); allColumns.addAll(coveredColumns); @@ -510,8 +518,7 @@ private RowKeyMetaData newRowKeyMetaData(int capacity) { public int getEstimatedByteSize() { int size = WritableUtils.getVIntSize(nIndexSaltBuckets); size += WritableUtils.getVIntSize(indexedColumns.size()); - for (int i = 0; i < indexedColumns.size(); i++) { - ColumnReference ref = indexedColumns.get(i); + for (ColumnReference ref : indexedColumns) { size += WritableUtils.getVIntSize(ref.getFamily().length); size += ref.getFamily().length; size += WritableUtils.getVIntSize(ref.getQualifier().length); @@ -520,8 +527,7 @@ public int getEstimatedByteSize() { size += indexedColumnTypes.size(); size += indexedColumnByteSizes.size(); size += WritableUtils.getVIntSize(coveredColumns.size()); - for (int i = 0; i < coveredColumns.size(); i++) { - ColumnReference ref = coveredColumns.get(i); + for (ColumnReference ref : coveredColumns) { size += WritableUtils.getVIntSize(ref.getFamily().length); size += ref.getFamily().length; size += WritableUtils.getVIntSize(ref.getQualifier().length); @@ -536,8 +542,7 @@ public int getEstimatedByteSize() { public void write(DataOutput output) throws IOException { WritableUtils.writeVInt(output, nIndexSaltBuckets); WritableUtils.writeVInt(output, indexedColumns.size()); - for (int i = 0; i < indexedColumns.size(); i++) { - ColumnReference ref = indexedColumns.get(i); + for (ColumnReference ref : indexedColumns) { Bytes.writeByteArray(output, ref.getFamily()); Bytes.writeByteArray(output, ref.getQualifier()); } @@ -550,8 +555,7 @@ public void write(DataOutput output) throws IOException { WritableUtils.writeVInt(output, byteSize == null ? 0 : byteSize); } WritableUtils.writeVInt(output, coveredColumns.size()); - for (int i = 0; i < coveredColumns.size(); i++) { - ColumnReference ref = coveredColumns.get(i); + for (ColumnReference ref : coveredColumns) { Bytes.writeByteArray(output, ref.getFamily()); Bytes.writeByteArray(output, ref.getQualifier()); } diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java index ced63eaa..92cd7656 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java @@ -63,8 +63,7 @@ public void batchStarted(MiniBatchOperationInProgress> m // as the empty key value column (which we use to detect a delete of the entire row). for (int i = 0; i < maintainers.size(); i++) { IndexMaintainer maintainer = maintainers.get(i); - for (int j = 0; j < maintainer.getAllColumns().size(); j++) { - ColumnReference ref = maintainer.getAllColumns().get(j); + for (ColumnReference ref : maintainer.getAllColumns()) { scan.addFamily(ref.getFamily()); } } From 6c2355fb2b1e30bc115a07648e490e02af11afd5 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 27 Sep 2013 17:19:35 -0700 Subject: [PATCH 085/102] Run UPSERT SELECT statement on client if table is marked as having immutable rows --- .../java/com/salesforce/phoenix/compile/UpsertCompiler.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/compile/UpsertCompiler.java b/src/main/java/com/salesforce/phoenix/compile/UpsertCompiler.java index 68d0ba18..d3572fc3 100644 --- a/src/main/java/com/salesforce/phoenix/compile/UpsertCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/UpsertCompiler.java @@ -272,11 +272,13 @@ public MutationPlan compile(UpsertStatement upsert, List binds) throws S * 1) the into table matches from table * 2) the select query isn't doing aggregation * 3) autoCommit is on - * 4) no limit clause + * 4) the table is not immutable, as the client is the one that figures out the additional + * puts for index tables. + * 5) no limit clause * Otherwise, run the query to pull the data from the server * and populate the MutationState (upto a limit). */ - runOnServer = sameTable && isAutoCommit && !select.isAggregate() && !select.isDistinct() && select.getLimit() == null && table.getBucketNum() == null; + runOnServer = sameTable && isAutoCommit && !table.isImmutableRows() && !select.isAggregate() && !select.isDistinct() && select.getLimit() == null && table.getBucketNum() == null; ParallelIteratorFactory parallelIteratorFactory; // TODO: once MutationState is thread safe, then when auto commit is off, we can still run in parallel if (select.isAggregate() || select.isDistinct() || select.getLimit() != null) { From 511598bf4f2de0da0d1d8f642288d2bcb6861113 Mon Sep 17 00:00:00 2001 From: maryannxue Date: Sat, 28 Sep 2013 14:16:53 -0400 Subject: [PATCH 086/102] add HashJoinTest --- .../phoenix/end2end/HashJoinTest.java | 307 ++++++++++++++++++ .../salesforce/phoenix/query/BaseTest.java | 26 ++ .../com/salesforce/phoenix/util/TestUtil.java | 4 + 3 files changed, 337 insertions(+) create mode 100644 src/test/java/com/salesforce/phoenix/end2end/HashJoinTest.java diff --git a/src/test/java/com/salesforce/phoenix/end2end/HashJoinTest.java b/src/test/java/com/salesforce/phoenix/end2end/HashJoinTest.java new file mode 100644 index 00000000..2a8ca03b --- /dev/null +++ b/src/test/java/com/salesforce/phoenix/end2end/HashJoinTest.java @@ -0,0 +1,307 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.phoenix.end2end; + +import static com.salesforce.phoenix.util.TestUtil.*; +import static org.junit.Assert.*; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.Properties; + +import org.junit.Test; + +import com.salesforce.phoenix.util.PhoenixRuntime; + +public abstract class HashJoinTest extends BaseClientMangedTimeTest { + + private void initMetaInfoTableValues(Long ts) throws Exception { + ensureTableCreated(getUrl(), JOIN_CUSTOMER_TABLE); + ensureTableCreated(getUrl(), JOIN_ITEM_TABLE); + ensureTableCreated(getUrl(), JOIN_SUPPLIER_TABLE); + + Properties props = new Properties(); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, ts.toString()); + Connection conn = DriverManager.getConnection(getUrl(), props); + try { + // Insert into customer table + PreparedStatement stmt = conn.prepareStatement( + "upsert into " + JOIN_CUSTOMER_TABLE + + " (CUSTOMER_ID, " + + " NAME, " + + " PHONE, " + + " ADDRESS) " + + "values (?, ?, ?, ?)"); + stmt.setString(1, "0000000001"); + stmt.setString(2, "C1"); + stmt.setString(3, "999-999-1111"); + stmt.setString(4, "101 XXX Street"); + stmt.execute(); + + stmt.setString(1, "0000000002"); + stmt.setString(2, "C2"); + stmt.setString(3, "999-999-2222"); + stmt.setString(4, "202 XXX Street"); + stmt.execute(); + + stmt.setString(1, "0000000003"); + stmt.setString(2, "C3"); + stmt.setString(3, "999-999-3333"); + stmt.setString(4, "303 XXX Street"); + stmt.execute(); + + stmt.setString(1, "0000000004"); + stmt.setString(2, "C4"); + stmt.setString(3, "999-999-4444"); + stmt.setString(4, "404 XXX Street"); + stmt.execute(); + + stmt.setString(1, "0000000005"); + stmt.setString(2, "C5"); + stmt.setString(3, "999-999-5555"); + stmt.setString(4, "505 XXX Street"); + stmt.execute(); + + stmt.setString(1, "0000000006"); + stmt.setString(2, "C6"); + stmt.setString(3, "999-999-6666"); + stmt.setString(4, "606 XXX Street"); + stmt.execute(); + + // Insert into item table + stmt = conn.prepareStatement( + "upsert into " + JOIN_ITEM_TABLE + + " (ITEM_ID, " + + " NAME, " + + " PRICE, " + + " SUPPLIER_ID, " + + " DESCRIPTION) " + + "values (?, ?, ?, ?)"); + stmt.setString(1, "0000000001"); + stmt.setString(2, "T1"); + stmt.setInt(3, 100); + stmt.setString(4, "0000000001"); + stmt.setString(5, "Item T1"); + stmt.execute(); + + stmt.setString(1, "0000000002"); + stmt.setString(2, "T2"); + stmt.setInt(3, 200); + stmt.setString(4, "0000000001"); + stmt.setString(5, "Item T2"); + stmt.execute(); + + stmt.setString(1, "0000000003"); + stmt.setString(2, "T3"); + stmt.setInt(3, 300); + stmt.setString(4, "0000000002"); + stmt.setString(5, "Item T3"); + stmt.execute(); + + stmt.setString(1, "0000000004"); + stmt.setString(2, "T4"); + stmt.setInt(3, 400); + stmt.setString(4, "0000000002"); + stmt.setString(5, "Item T4"); + stmt.execute(); + + stmt.setString(1, "0000000005"); + stmt.setString(2, "T5"); + stmt.setInt(3, 500); + stmt.setString(4, "0000000005"); + stmt.setString(5, "Item T5"); + stmt.execute(); + + stmt.setString(1, "0000000006"); + stmt.setString(2, "T6"); + stmt.setInt(3, 600); + stmt.setString(4, "0000000006"); + stmt.setString(5, "Item T6"); + stmt.execute(); + + stmt.setString(1, "invalid001"); + stmt.setString(2, "INVALID-1"); + stmt.setInt(3, 0); + stmt.setString(4, "0000000000"); + stmt.setString(5, "Invalid item for join test"); + stmt.execute(); + + // Insert into supplier table + stmt = conn.prepareStatement( + "upsert into " + JOIN_SUPPLIER_TABLE + + " (SUPPLIER_ID, " + + " NAME, " + + " PHONE, " + + " ADDRESS) " + + "values (?, ?, ?, ?)"); + stmt.setString(1, "0000000001"); + stmt.setString(2, "S1"); + stmt.setString(3, "888-888-1111"); + stmt.setString(4, "101 YYY Street"); + stmt.execute(); + + stmt.setString(1, "0000000002"); + stmt.setString(2, "S2"); + stmt.setString(3, "888-888-2222"); + stmt.setString(4, "202 YYY Street"); + stmt.execute(); + + stmt.setString(1, "0000000003"); + stmt.setString(2, "S3"); + stmt.setString(3, "888-888-3333"); + stmt.setString(4, "303 YYY Street"); + stmt.execute(); + + stmt.setString(1, "0000000004"); + stmt.setString(2, "S4"); + stmt.setString(3, "888-888-4444"); + stmt.setString(4, "404 YYY Street"); + stmt.execute(); + + stmt.setString(1, "0000000005"); + stmt.setString(2, "S5"); + stmt.setString(3, "888-888-5555"); + stmt.setString(4, "505 YYY Street"); + stmt.execute(); + + stmt.setString(1, "0000000006"); + stmt.setString(2, "S6"); + stmt.setString(3, "888-888-6666"); + stmt.setString(4, "606 YYY Street"); + stmt.execute(); + + conn.commit(); + } finally { + conn.close(); + } + } + + @Test + public void testInnerJoin() throws Exception { + long ts = nextTimestamp(); + initMetaInfoTableValues(ts); + String query = "SELECT item.item_id, item.name, supp.supplier_id, supp.name FROM " + JOIN_ITEM_TABLE + " item INNER JOIN " + JOIN_SUPPLIER_TABLE + " supp ON item.supplier_id = supp.supplier_id"; + Properties props = new Properties(TEST_PROPERTIES); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 + Connection conn = DriverManager.getConnection(PHOENIX_JDBC_URL, props); + try { + PreparedStatement statement = conn.prepareStatement(query); + ResultSet rs = statement.executeQuery(); + assertTrue (rs.next()); + assertEquals(rs.getString(1), "0000000001"); + assertEquals(rs.getString(2), "T1"); + assertEquals(rs.getString(3), "0000000001"); + assertEquals(rs.getString(4), "S1"); + assertTrue (rs.next()); + assertEquals(rs.getString(1), "0000000002"); + assertEquals(rs.getString(2), "T2"); + assertEquals(rs.getString(3), "0000000001"); + assertEquals(rs.getString(4), "S1"); + assertTrue (rs.next()); + assertEquals(rs.getString(1), "0000000003"); + assertEquals(rs.getString(2), "T3"); + assertEquals(rs.getString(3), "0000000002"); + assertEquals(rs.getString(4), "S2"); + assertTrue (rs.next()); + assertEquals(rs.getString(1), "0000000004"); + assertEquals(rs.getString(2), "T4"); + assertEquals(rs.getString(3), "0000000002"); + assertEquals(rs.getString(4), "S2"); + assertTrue (rs.next()); + assertEquals(rs.getString(1), "0000000005"); + assertEquals(rs.getString(2), "T5"); + assertEquals(rs.getString(3), "0000000005"); + assertEquals(rs.getString(4), "S5"); + assertTrue (rs.next()); + assertEquals(rs.getString(1), "0000000006"); + assertEquals(rs.getString(2), "T6"); + assertEquals(rs.getString(3), "0000000006"); + assertEquals(rs.getString(4), "S6"); + + assertFalse(rs.next()); + } finally { + conn.close(); + } + } + + @Test + public void testLeftJoin() throws Exception { + long ts = nextTimestamp(); + initMetaInfoTableValues(ts); + String query = "SELECT item.item_id, item.name, supp.supplier_id, supp.name FROM " + JOIN_ITEM_TABLE + " item LEFT JOIN " + JOIN_SUPPLIER_TABLE + " supp ON item.supplier_id = supp.supplier_id"; + Properties props = new Properties(TEST_PROPERTIES); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 + Connection conn = DriverManager.getConnection(PHOENIX_JDBC_URL, props); + try { + PreparedStatement statement = conn.prepareStatement(query); + ResultSet rs = statement.executeQuery(); + assertTrue (rs.next()); + assertEquals(rs.getString(1), "0000000001"); + assertEquals(rs.getString(2), "T1"); + assertEquals(rs.getString(3), "0000000001"); + assertEquals(rs.getString(4), "S1"); + assertTrue (rs.next()); + assertEquals(rs.getString(1), "0000000002"); + assertEquals(rs.getString(2), "T2"); + assertEquals(rs.getString(3), "0000000001"); + assertEquals(rs.getString(4), "S1"); + assertTrue (rs.next()); + assertEquals(rs.getString(1), "0000000003"); + assertEquals(rs.getString(2), "T3"); + assertEquals(rs.getString(3), "0000000002"); + assertEquals(rs.getString(4), "S2"); + assertTrue (rs.next()); + assertEquals(rs.getString(1), "0000000004"); + assertEquals(rs.getString(2), "T4"); + assertEquals(rs.getString(3), "0000000002"); + assertEquals(rs.getString(4), "S2"); + assertTrue (rs.next()); + assertEquals(rs.getString(1), "0000000005"); + assertEquals(rs.getString(2), "T5"); + assertEquals(rs.getString(3), "0000000005"); + assertEquals(rs.getString(4), "S5"); + assertTrue (rs.next()); + assertEquals(rs.getString(1), "0000000006"); + assertEquals(rs.getString(2), "T6"); + assertEquals(rs.getString(3), "0000000006"); + assertEquals(rs.getString(4), "S6"); + assertTrue (rs.next()); + assertEquals(rs.getString(1), "invalid001"); + assertEquals(rs.getString(2), "INVALID-1"); + assertNull(rs.getString(3)); + assertNull(rs.getString(4)); + + assertFalse(rs.next()); + } finally { + conn.close(); + } + } + +} diff --git a/src/test/java/com/salesforce/phoenix/query/BaseTest.java b/src/test/java/com/salesforce/phoenix/query/BaseTest.java index 013d9308..b3b53dae 100644 --- a/src/test/java/com/salesforce/phoenix/query/BaseTest.java +++ b/src/test/java/com/salesforce/phoenix/query/BaseTest.java @@ -242,6 +242,32 @@ public abstract class BaseTest { "IMMUTABLE_ROWS=true"); builder.put("SumDoubleTest","create table SumDoubleTest" + " (id varchar not null primary key, d DOUBLE, f FLOAT, ud UNSIGNED_DOUBLE, uf UNSIGNED_FLOAT, i integer, de decimal)"); + builder.put(JOIN_ORDER_TABLE, "create table " + JOIN_ORDER_TABLE + + " (order_id char(15) not null, " + + " customer_id char(10) not null, " + + " item_id char(10) not null, " + + " quantity integer not null, " + + " date date not null " + + " CONSTRAINT pk PRIMARY KEY (order_id))"); + builder.put(JOIN_CUSTOMER_TABLE, "create table " + JOIN_CUSTOMER_TABLE + + " (customer_id char(10) not null, " + + " name varchar not null, " + + " phone char(12), " + + " address varchar " + + " CONSTRAINT pk PRIMARY KEY (customer_id))"); + builder.put(JOIN_ITEM_TABLE, "create table " + JOIN_ITEM_TABLE + + " (item_id char(10) not null, " + + " name varchar not null, " + + " price integer not null, " + + " supplier_id char(10) not null, " + + " description varchar " + + " CONSTRAINT pk PRIMARY KEY (item_id))"); + builder.put(JOIN_SUPPLIER_TABLE, "create table " + JOIN_SUPPLIER_TABLE + + " (supplier_id char(10) not null, " + + " name varchar not null, " + + " phone char(12), " + + " address varchar " + + " CONSTRAINT pk PRIMARY KEY (supplier_id))"); tableDDLMap = builder.build(); } diff --git a/src/test/java/com/salesforce/phoenix/util/TestUtil.java b/src/test/java/com/salesforce/phoenix/util/TestUtil.java index 533c74e1..51e1cd21 100644 --- a/src/test/java/com/salesforce/phoenix/util/TestUtil.java +++ b/src/test/java/com/salesforce/phoenix/util/TestUtil.java @@ -117,6 +117,10 @@ private TestUtil() { public static final String TABLE_WITH_SALTING = "TABLE_WITH_SALTING"; public static final String INDEX_DATA_SCHEMA = "INDEX_TEST"; public static final String INDEX_DATA_TABLE = "INDEX_DATA_TABLE"; + public static final String JOIN_ORDER_TABLE = "JOIN_ORDER_TABLE"; + public static final String JOIN_CUSTOMER_TABLE = "JOIN_CUSTOMER_TABLE"; + public static final String JOIN_ITEM_TABLE = "JOIN_ITEM_TABLE"; + public static final String JOIN_SUPPLIER_TABLE = "JOIN_SUPPLIER_TABLE"; public static final Properties TEST_PROPERTIES = new Properties(); From a002c71bfdeb533d943d4516a5fd76f8edcc1b10 Mon Sep 17 00:00:00 2001 From: mujtabachohan Date: Mon, 30 Sep 2013 12:18:32 -0700 Subject: [PATCH 087/102] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 69da9cc4..b7467f3e 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ Currently, Phoenix hosts its own maven repository in github. This is done for co com.salesforce phoenix - 2.0.0 + 2.0.2 ... From dbc120c977ae56d17e0bb49a250e124a63ad5c2c Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Fri, 27 Sep 2013 13:44:25 -0700 Subject: [PATCH 088/102] Late-replaying index writes from the WAL. When WAL recovery occurs, we need to ensure that we can write to the index table. If the index region we were targeting lived on the same server as the primary region, it will likely be involved in the same log splitting and recovery process. This means, when we replay the writes into the primary table, the index table/region we would write to is probably not up yet. In 0.96 this is fixed by allowing regions to take writes before reads during WAL replay. We are only on 0.94 (and its a huge effort to backport that feature), so instead we can get around the issue by buffering the failed writes and replaying them later (in the postOpen hook). --- .../com/salesforce/hbase/index/Indexer.java | 81 +++- .../CoveredColumnIndexSpecifierBuilder.java | 10 +- .../covered/example/CoveredColumnIndexer.java | 9 +- .../CannotReachSingleIndexException.java} | 8 +- .../index/exception/IndexWriteException.java | 51 +++ .../MutliIndexWriteFailureException.java | 54 +++ .../hbase/index/util/IndexManagementUtil.java | 24 ++ .../hbase/index/write/IndexCommitter.java | 4 +- .../hbase/index/write/IndexWriter.java | 78 ++-- .../write/ParallelWriterIndexCommitter.java | 33 +- .../recovery/PerRegionIndexWriteCache.java | 73 ++++ .../recovery/StoreFailuresInCachePolicy.java | 83 ++++ .../TrackingParallelWriterIndexCommitter.java | 266 +++++++++++++ .../regionserver/wal/IndexedHLogReader.java | 11 +- .../hbase/index/IndexTestingUtils.java | 8 + .../hbase/index/write/TestIndexWriter.java | 7 +- .../index/write/TestWALRecoveryCaching.java | 375 ++++++++++++++++++ .../TestPerRegionIndexWriteCache.java | 178 +++++++++ 18 files changed, 1294 insertions(+), 59 deletions(-) rename src/main/java/com/salesforce/hbase/index/{CannotReachIndexException.java => exception/CannotReachSingleIndexException.java} (90%) create mode 100644 src/main/java/com/salesforce/hbase/index/exception/IndexWriteException.java create mode 100644 src/main/java/com/salesforce/hbase/index/exception/MutliIndexWriteFailureException.java create mode 100644 src/main/java/com/salesforce/hbase/index/write/recovery/PerRegionIndexWriteCache.java create mode 100644 src/main/java/com/salesforce/hbase/index/write/recovery/StoreFailuresInCachePolicy.java create mode 100644 src/main/java/com/salesforce/hbase/index/write/recovery/TrackingParallelWriterIndexCommitter.java create mode 100644 src/test/java/com/salesforce/hbase/index/write/TestWALRecoveryCaching.java create mode 100644 src/test/java/com/salesforce/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java diff --git a/src/main/java/com/salesforce/hbase/index/Indexer.java b/src/main/java/com/salesforce/hbase/index/Indexer.java index b807d7ad..c9405531 100644 --- a/src/main/java/com/salesforce/hbase/index/Indexer.java +++ b/src/main/java/com/salesforce/hbase/index/Indexer.java @@ -65,10 +65,17 @@ import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.util.Pair; +import com.google.common.collect.Multimap; import com.salesforce.hbase.index.builder.IndexBuilder; +import com.salesforce.hbase.index.exception.IndexWriteException; +import com.salesforce.hbase.index.table.HTableInterfaceReference; import com.salesforce.hbase.index.util.ImmutableBytesPtr; import com.salesforce.hbase.index.wal.IndexedKeyValue; +import com.salesforce.hbase.index.write.IndexFailurePolicy; import com.salesforce.hbase.index.write.IndexWriter; +import com.salesforce.hbase.index.write.recovery.PerRegionIndexWriteCache; +import com.salesforce.hbase.index.write.recovery.StoreFailuresInCachePolicy; +import com.salesforce.hbase.index.write.recovery.TrackingParallelWriterIndexCommitter; /** * Do all the work of managing index updates from a single coprocessor. All Puts/Delets are passed @@ -106,6 +113,8 @@ public class Indexer extends BaseRegionObserver { */ public static final String CHECK_VERSION_CONF_KEY = "com.saleforce.hbase.index.checkversion"; + private static final String INDEX_RECOVERY_FAILURE_POLICY_KEY = "com.salesforce.hbase.index.recovery.failurepolicy"; + /** * Marker {@link KeyValue} to indicate that we are doing a batch operation. Needed because the * coprocessor framework throws away the WALEdit from the prePut/preDelete hooks when checking a @@ -114,6 +123,21 @@ public class Indexer extends BaseRegionObserver { */ private static KeyValue BATCH_MARKER = new KeyValue(); + /** + * cache the failed updates to the various regions. Used for making the WAL recovery mechanisms + * more robust in the face of recoverying index regions that were on the same server as the + * primary table region + */ + private PerRegionIndexWriteCache failedIndexEdits = new PerRegionIndexWriteCache(); + + /** + * IndexWriter for writing the recovered index edits. Separate from the main indexer since we need + * different write/failure policies + */ + private IndexWriter recoveryWriter; + + public static String RecoveryFailurePolicyKeyForTesting = INDEX_RECOVERY_FAILURE_POLICY_KEY; + @Override public void start(CoprocessorEnvironment e) throws IOException { @@ -149,13 +173,35 @@ public void start(CoprocessorEnvironment e) throws IOException { // add a synchronizer so we don't archive a WAL that we need log.registerWALActionsListener(new IndexLogRollSynchronizer(INDEX_READ_WRITE_LOCK.writeLock())); - // and setup the actual index writer + // setup the actual index writer this.writer = new IndexWriter(env); + + // setup the recovery writer that does retries on the failed edits + TrackingParallelWriterIndexCommitter recoveryCommmiter = + new TrackingParallelWriterIndexCommitter(); + + try { + // get the specified failure policy. We only ever override it in tests, but we need to do it + // here + Class policyClass = conf.getClass( + INDEX_RECOVERY_FAILURE_POLICY_KEY, + StoreFailuresInCachePolicy.class, IndexFailurePolicy.class); + IndexFailurePolicy policy = policyClass.getConstructor(PerRegionIndexWriteCache.class) + .newInstance(failedIndexEdits); + LOG.debug("Setting up recovery writter with committer: " + recoveryCommmiter.getClass() + + " and failure policy: " + policy.getClass()); + recoveryWriter = new IndexWriter(recoveryCommmiter, policy, env); + } catch (Exception ex) { + throw new IOException("Could not instantiate recovery failure policy!", ex); + } + } @Override public void stop(CoprocessorEnvironment e) throws IOException { - this.writer.stop("Indexer is being stopped"); + String msg = "Indexer is being stopped"; + this.writer.stop(msg); + this.recoveryWriter.stop(msg); } @Override @@ -266,7 +312,6 @@ public MultiMutation(ImmutableBytesPtr rowkey, boolean writeToWal) { /** * @param stored */ - @SuppressWarnings("deprecation") public void addAll(Mutation stored) { // add all the kvs for (Entry> kvs : stored.getFamilyMap().entrySet()) { @@ -339,7 +384,7 @@ private boolean doPre(Collection> indexUpdates, final WAL try { this.writer.write(indexUpdates); return false; - } catch (CannotReachIndexException e) { + } catch (IndexWriteException e) { LOG.error("Failed to update index with entries:" + indexUpdates, e); throw new IOException(e); } @@ -454,11 +499,37 @@ private Collection> extractIndexUpdate(WALEdit edit) { return indexUpdates; } + @Override + public void postOpen(final ObserverContext c) { + Multimap updates = failedIndexEdits.getEdits(c.getEnvironment().getRegion()); + + LOG.info("Found some outstanding index updates that didn't succeed during" + + " WAL replay - attempting to replay now."); + //if we have no pending edits to complete, then we are done + if (updates == null || updates.size() == 0) { + return; + } + + // do the usual writer stuff, killing the server again, if we can't manage to make the index + // writes succeed again + writer.writeAndKillYourselfOnFailure(updates); + } + @Override public void preWALRestore(ObserverContext env, HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { + // TODO check the regions in transition. If the server on which the region lives is this one, + // then we should rety that write later in postOpen. + // we might be able to get even smarter here and pre-split the edits that are server-local + // into their own recovered.edits file. This then lets us do a straightforward recovery of each + // region (and more efficiently as we aren't writing quite as hectically from this one place). + + /* + * Basically, we let the index regions recover for a little while long before retrying in the + * hopes they come up before the primary table finishes. + */ Collection> indexUpdates = extractIndexUpdate(logEdit); - writer.writeAndKillYourselfOnFailure(indexUpdates); + recoveryWriter.writeAndKillYourselfOnFailure(indexUpdates); } /** diff --git a/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexSpecifierBuilder.java b/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexSpecifierBuilder.java index e24868f7..36b9ae66 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexSpecifierBuilder.java +++ b/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexSpecifierBuilder.java @@ -34,6 +34,7 @@ public class CoveredColumnIndexSpecifierBuilder { // private static final String INDEX_GROUP_FULLY_COVERED = ".covered"; List groups = new ArrayList(); + private Map specs = new HashMap(); /** * Add a group of columns to index @@ -66,7 +67,6 @@ public void reset() { } Map convertToMap() { - Map specs = new HashMap(); int total = this.groups.size(); // hbase.index.covered.groups = i specs.put(INDEX_GROUPS_COUNT_KEY, Integer.toString(total)); @@ -150,4 +150,12 @@ static List getColumns(Configuration conf) { } return columns; } + + /** + * @param recoveryFailurePolicyKeyForTesting + * @param class1 + */ + public void addArbitraryConfigForTesting(String key, String value) { + this.specs.put(key, value); + } } \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexer.java b/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexer.java index 299f3b36..107bc9bf 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexer.java +++ b/src/main/java/com/salesforce/hbase/index/covered/example/CoveredColumnIndexer.java @@ -82,7 +82,14 @@ public class CoveredColumnIndexer extends CoveredColumnsIndexBuilder { * @throws IOException */ public static void createIndexTable(HBaseAdmin admin, String indexTable) throws IOException { - HTableDescriptor index = new HTableDescriptor(indexTable); + createIndexTable(admin, new HTableDescriptor(indexTable)); + } + + /** + * @param admin to create the table + * @param index descriptor to update before creating table + */ + public static void createIndexTable(HBaseAdmin admin, HTableDescriptor index) throws IOException { HColumnDescriptor col = new HColumnDescriptor(CoveredColumnIndexCodec.INDEX_ROW_COLUMN_FAMILY); // ensure that we can 'see past' delete markers when doing scans diff --git a/src/main/java/com/salesforce/hbase/index/CannotReachIndexException.java b/src/main/java/com/salesforce/hbase/index/exception/CannotReachSingleIndexException.java similarity index 90% rename from src/main/java/com/salesforce/hbase/index/CannotReachIndexException.java rename to src/main/java/com/salesforce/hbase/index/exception/CannotReachSingleIndexException.java index 895e4a0a..c6658951 100644 --- a/src/main/java/com/salesforce/hbase/index/CannotReachIndexException.java +++ b/src/main/java/com/salesforce/hbase/index/exception/CannotReachSingleIndexException.java @@ -25,7 +25,7 @@ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************/ -package com.salesforce.hbase.index; +package com.salesforce.hbase.index.exception; import java.util.List; @@ -35,7 +35,7 @@ * Exception thrown if we cannot successfully write to an index table. */ @SuppressWarnings("serial") -public class CannotReachIndexException extends Exception { +public class CannotReachSingleIndexException extends IndexWriteException { private String table; @@ -44,7 +44,7 @@ public class CannotReachIndexException extends Exception { * @param msg more description of what happened * @param cause original cause */ - public CannotReachIndexException(String msg, Throwable cause) { + public CannotReachSingleIndexException(String msg, Throwable cause) { super(msg, cause); } @@ -54,7 +54,7 @@ public CannotReachIndexException(String msg, Throwable cause) { * @param mutations mutations that were attempted * @param cause underlying reason for the failure */ - public CannotReachIndexException(String targetTableName, List mutations, Exception cause) { + public CannotReachSingleIndexException(String targetTableName, List mutations, Exception cause) { super("Failed to make index update:\n\t table: " + targetTableName + "\n\t edits: " + mutations + "\n\tcause: " + cause == null ? "UNKNOWN" : cause.getMessage(), cause); this.table = targetTableName; diff --git a/src/main/java/com/salesforce/hbase/index/exception/IndexWriteException.java b/src/main/java/com/salesforce/hbase/index/exception/IndexWriteException.java new file mode 100644 index 00000000..85ef6d62 --- /dev/null +++ b/src/main/java/com/salesforce/hbase/index/exception/IndexWriteException.java @@ -0,0 +1,51 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index.exception; + +/** + * Generic {@link Exception} that an index write has failed + */ +@SuppressWarnings("serial") +public class IndexWriteException extends Exception { + + public IndexWriteException() { + super(); + } + + public IndexWriteException(String message, Throwable cause) { + super(message, cause); + } + + public IndexWriteException(String message) { + super(message); + } + + public IndexWriteException(Throwable cause) { + super(cause); + } +} \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/exception/MutliIndexWriteFailureException.java b/src/main/java/com/salesforce/hbase/index/exception/MutliIndexWriteFailureException.java new file mode 100644 index 00000000..abeb0a2a --- /dev/null +++ b/src/main/java/com/salesforce/hbase/index/exception/MutliIndexWriteFailureException.java @@ -0,0 +1,54 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index.exception; + +import java.util.List; + +import com.salesforce.hbase.index.table.HTableInterfaceReference; + +/** + * Indicate a failure to write to multiple index tables. + */ +@SuppressWarnings("serial") +public class MutliIndexWriteFailureException extends IndexWriteException { + + private List failures; + + /** + * @param failures the tables to which the index write did not succeed + */ + public MutliIndexWriteFailureException(List failures) { + super("Failed to write to multiple index tables"); + this.failures = failures; + + } + + public List getFailedTables() { + return this.failures; + } +} \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java b/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java index 4205d50c..64d78090 100644 --- a/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java +++ b/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java @@ -32,8 +32,13 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.regionserver.wal.IndexedHLogReader; +import org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec; +import org.apache.hadoop.hbase.regionserver.wal.WALEditCodec; +import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.salesforce.hbase.index.ValueGetter; import com.salesforce.hbase.index.covered.data.LazyValueGetter; @@ -49,6 +54,25 @@ private IndexManagementUtil() { // private ctor for util classes } + public static String HLOG_READER_IMPL_KEY = "hbase.regionserver.hlog.reader.impl"; + + public static void ensureMutableIndexingCorrectlyConfigured(Configuration conf) + throws IllegalStateException { + // ensure the WALEditCodec is correct + Preconditions + .checkState( + conf.getClass(WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY, IndexedWALEditCodec.class) == IndexedWALEditCodec.class, + IndexedWALEditCodec.class.getName() + + " was not installed. You need to install it in hbase-site.xml under " + + WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY); + // ensure the Hlog Reader is correct + Preconditions.checkState( + conf.getClass(HLOG_READER_IMPL_KEY, IndexedHLogReader.class) == IndexedHLogReader.class, + IndexedHLogReader.class.getName() + + " was not installed. You need to install it in hbase-site.xml under " + + HLOG_READER_IMPL_KEY); + } + public static ValueGetter createGetterFromKeyValues(Collection pendingUpdates) { final Map valueMap = Maps.newHashMapWithExpectedSize(pendingUpdates.size()); diff --git a/src/main/java/com/salesforce/hbase/index/write/IndexCommitter.java b/src/main/java/com/salesforce/hbase/index/write/IndexCommitter.java index a763f329..11b2c6c6 100644 --- a/src/main/java/com/salesforce/hbase/index/write/IndexCommitter.java +++ b/src/main/java/com/salesforce/hbase/index/write/IndexCommitter.java @@ -32,7 +32,7 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import com.google.common.collect.Multimap; -import com.salesforce.hbase.index.CannotReachIndexException; +import com.salesforce.hbase.index.exception.IndexWriteException; import com.salesforce.hbase.index.table.HTableInterfaceReference; /** @@ -43,5 +43,5 @@ public interface IndexCommitter extends Stoppable { void setup(IndexWriter parent, RegionCoprocessorEnvironment env); public void write(Multimap toWrite) - throws CannotReachIndexException; + throws IndexWriteException; } \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/write/IndexWriter.java b/src/main/java/com/salesforce/hbase/index/write/IndexWriter.java index 1368aecb..3632d77c 100644 --- a/src/main/java/com/salesforce/hbase/index/write/IndexWriter.java +++ b/src/main/java/com/salesforce/hbase/index/write/IndexWriter.java @@ -43,7 +43,7 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; -import com.salesforce.hbase.index.CannotReachIndexException; +import com.salesforce.hbase.index.exception.IndexWriteException; import com.salesforce.hbase.index.table.HTableInterfaceReference; import com.salesforce.hbase.index.util.ImmutableBytesPtr; @@ -68,12 +68,10 @@ public class IndexWriter implements Stoppable { * instantiated */ public IndexWriter(RegionCoprocessorEnvironment env) throws IOException { - this(getCommitter(env), getFailurePolicy(env)); - this.writer.setup(this, env); - this.failurePolicy.setup(this, env); + this(getCommitter(env), getFailurePolicy(env), env); } - static IndexCommitter getCommitter(RegionCoprocessorEnvironment env) throws IOException { + public static IndexCommitter getCommitter(RegionCoprocessorEnvironment env) throws IOException { Configuration conf = env.getConfiguration(); try { IndexCommitter committer = @@ -87,7 +85,7 @@ static IndexCommitter getCommitter(RegionCoprocessorEnvironment env) throws IOEx } } - static IndexFailurePolicy getFailurePolicy(RegionCoprocessorEnvironment env) + public static IndexFailurePolicy getFailurePolicy(RegionCoprocessorEnvironment env) throws IOException { Configuration conf = env.getConfiguration(); try { @@ -103,35 +101,55 @@ static IndexFailurePolicy getFailurePolicy(RegionCoprocessorEnvironment env) } /** - * Exposed for TESTING! Does no setup of the {@link IndexFailurePolicy} or {@link IndexCommitter}. + * Directly specify the {@link IndexCommitter} and {@link IndexFailurePolicy}. Both are expected + * to be fully setup before calling. + * @param committer + * @param policy + * @param env + */ + public IndexWriter(IndexCommitter committer, IndexFailurePolicy policy, + RegionCoprocessorEnvironment env) { + this(committer, policy); + this.writer.setup(this, env); + this.failurePolicy.setup(this, env); + } + + /** + * Create an {@link IndexWriter} with an already setup {@link IndexCommitter} and + * {@link IndexFailurePolicy}. + * @param committer to write updates + * @param policy to handle failures */ IndexWriter(IndexCommitter committer, IndexFailurePolicy policy) { this.writer = committer; this.failurePolicy = policy; } + /** * Write the mutations to their respective table. *

    * This method is blocking and could potentially cause the writer to block for a long time as we - * write the index updates. We only return when either: - *

      - *
    1. All index writes have returned, OR
      2. - *
    2. Any single index write has failed
      3. - *
    - * We attempt to quickly determine if any write has failed and not write to the remaining indexes - * to ensure a timely recovery of the failed index writes. + * write the index updates. When we return depends on the specified {@link IndexCommitter}. *

    - * If any of the index updates fails, we pass along the failure to the installed - * {@link IndexFailurePolicy}, which then decides how to handle the failure. By default, we use a - * {@link KillServerOnFailurePolicy}, which ensures that the server crashes when an index write - * fails, ensuring that we get WAL replay of the index edits. + * If update fails, we pass along the failure to the installed {@link IndexFailurePolicy}, which + * then decides how to handle the failure. By default, we use a {@link KillServerOnFailurePolicy}, + * which ensures that the server crashes when an index write fails, ensuring that we get WAL + * replay of the index edits. * @param indexUpdates Updates to write */ public void writeAndKillYourselfOnFailure(Collection> indexUpdates) { // convert the strings to htableinterfaces to which we can talk and group by TABLE Multimap toWrite = resolveTableReferences(indexUpdates); + writeAndKillYourselfOnFailure(toWrite); + } + + /** + * see {@link #writeAndKillYourselfOnFailure(Collection)}. + * @param toWrite + */ + public void writeAndKillYourselfOnFailure(Multimap toWrite) { try { - this.writer.write(toWrite); + write(toWrite); LOG.info("Done writing all index updates!"); } catch (Exception e) { this.failurePolicy.handleFailure(toWrite, e); @@ -144,23 +162,27 @@ public void writeAndKillYourselfOnFailure(Collection> ind * This method is blocking and could potentially cause the writer to block for a long time as we * write the index updates. We only return when either: *

      - *
    1. All index writes have returned, OR
      2. - *
    2. Any single index write has failed
      3. + *
    3. All index writes have returned, OR
      4. + *
    4. Any single index write has failed
      5. *
    * We attempt to quickly determine if any write has failed and not write to the remaining indexes * to ensure a timely recovery of the failed index writes. * @param toWrite Updates to write - * @throws CannotReachIndexException if we cannot successfully write a single index entry. We stop - * immediately on the first failed index write, rather than attempting all writes. + * @throws IndexWriteException if we cannot successfully write to the index. Whether or not we + * stop early depends on the {@link IndexCommitter}. */ - public void write(Collection> toWrite) throws CannotReachIndexException { - this.writer.write(resolveTableReferences(toWrite)); + public void write(Collection> toWrite) throws IndexWriteException { + write(resolveTableReferences(toWrite)); } - + /** + * see {@link #write(Collection)} + * @param toWrite + * @throws IndexWriteException + */ public void write(Multimap toWrite) - throws CannotReachIndexException { - + throws IndexWriteException { + this.writer.write(toWrite); } diff --git a/src/main/java/com/salesforce/hbase/index/write/ParallelWriterIndexCommitter.java b/src/main/java/com/salesforce/hbase/index/write/ParallelWriterIndexCommitter.java index dddd7b55..4e502d16 100644 --- a/src/main/java/com/salesforce/hbase/index/write/ParallelWriterIndexCommitter.java +++ b/src/main/java/com/salesforce/hbase/index/write/ParallelWriterIndexCommitter.java @@ -57,15 +57,22 @@ import com.google.common.collect.Multimap; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import com.salesforce.hbase.index.CannotReachIndexException; import com.salesforce.hbase.index.CapturingAbortable; +import com.salesforce.hbase.index.exception.CannotReachSingleIndexException; import com.salesforce.hbase.index.table.CachingHTableFactory; import com.salesforce.hbase.index.table.CoprocessorHTableFactory; import com.salesforce.hbase.index.table.HTableFactory; import com.salesforce.hbase.index.table.HTableInterfaceReference; /** - * + * Write index updates to the index tables in parallel. We attempt to early exit from the writes if + * any of the index updates fails. Completion is determined by the following criteria: * + *
      + *
    1. All index writes have returned, OR
      2. + *
    2. Any single index write has failed
      3. + *
    + * We attempt to quickly determine if any write has failed and not write to the remaining indexes to + * ensure a timely recovery of the failed index writes. */ public class ParallelWriterIndexCommitter implements IndexCommitter { @@ -101,7 +108,7 @@ void setup(HTableFactory factory, ExecutorService pool, Abortable abortable, Sto this.stopped = stop; } - private static HTableFactory getDefaultDelegateHTableFactory(CoprocessorEnvironment env) { + public static HTableFactory getDefaultDelegateHTableFactory(CoprocessorEnvironment env) { // create a simple delegate factory, setup the way we need Configuration conf = env.getConfiguration(); // only have one thread per table - all the writes are already batched per table. @@ -112,9 +119,9 @@ private static HTableFactory getDefaultDelegateHTableFactory(CoprocessorEnvironm /** * @param conf - * @return + * @return a thread pool based on the passed configuration whose threads are all daemon threads. */ - private static ThreadPoolExecutor getDefaultExecutor(Configuration conf) { + public static ThreadPoolExecutor getDefaultExecutor(Configuration conf) { int maxThreads = conf.getInt(NUM_CONCURRENT_INDEX_WRITER_THREADS_CONF_KEY, DEFAULT_CONCURRENT_INDEX_WRITER_THREADS); @@ -141,7 +148,7 @@ private static ThreadPoolExecutor getDefaultExecutor(Configuration conf) { @Override public void write(Multimap toWrite) - throws CannotReachIndexException { + throws CannotReachSingleIndexException { /* * This bit here is a little odd, so let's explain what's going on. Basically, we want to do the * writes in parallel to each index table, so each table gets its own task and is submitted to @@ -194,22 +201,22 @@ public Void call() throws Exception { HTableInterface table = factory.getTable(tableReference.get()); throwFailureIfDone(); table.batch(mutations); - } catch (CannotReachIndexException e) { + } catch (CannotReachSingleIndexException e) { throw e; } catch (IOException e) { - throw new CannotReachIndexException(tableReference.toString(), mutations, e); + throw new CannotReachSingleIndexException(tableReference.toString(), mutations, e); } catch (InterruptedException e) { // reset the interrupt status on the thread Thread.currentThread().interrupt(); - throw new CannotReachIndexException(tableReference.toString(), mutations, e); + throw new CannotReachSingleIndexException(tableReference.toString(), mutations, e); } return null; } - private void throwFailureIfDone() throws CannotReachIndexException { + private void throwFailureIfDone() throws CannotReachSingleIndexException { if (stopped.isStopped() || abortable.isAborted() || Thread.currentThread().isInterrupted()) { - throw new CannotReachIndexException( + throw new CannotReachSingleIndexException( "Pool closed, not attempting to write to the index!", null); } @@ -258,10 +265,10 @@ private void throwFailureIfDone() throws CannotReachIndexException { // propagate the failure up to the caller try { this.abortable.throwCauseIfAborted(); - } catch (CannotReachIndexException e) { + } catch (CannotReachSingleIndexException e) { throw e; } catch (Throwable e) { - throw new CannotReachIndexException("Got an abort notification while writing to the index!", + throw new CannotReachSingleIndexException("Got an abort notification while writing to the index!", e); } diff --git a/src/main/java/com/salesforce/hbase/index/write/recovery/PerRegionIndexWriteCache.java b/src/main/java/com/salesforce/hbase/index/write/recovery/PerRegionIndexWriteCache.java new file mode 100644 index 00000000..18b4c2f1 --- /dev/null +++ b/src/main/java/com/salesforce/hbase/index/write/recovery/PerRegionIndexWriteCache.java @@ -0,0 +1,73 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index.write.recovery; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.regionserver.HRegion; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; +import com.salesforce.hbase.index.table.HTableInterfaceReference; + + +public class PerRegionIndexWriteCache { + + private Map> cache = + new HashMap>(); + + + /** + * Get the edits for the current region. Removes the edits from the cache. To add them back, call + * {@link #addEdits(HRegion, HTableInterfaceReference, Collection)}. + * @param region + * @return Get the edits for the given region. Returns null if there are no pending edits + * for the region + */ + public Multimap getEdits(HRegion region) { + return cache.remove(region); + } + + /** + * @param region + * @param table + * @param collection + */ + public void addEdits(HRegion region, HTableInterfaceReference table, + Collection collection) { + Multimap edits = cache.get(region); + if (edits == null) { + edits = ArrayListMultimap. create(); + cache.put(region, edits); + } + edits.putAll(table, collection); + } +} \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/write/recovery/StoreFailuresInCachePolicy.java b/src/main/java/com/salesforce/hbase/index/write/recovery/StoreFailuresInCachePolicy.java new file mode 100644 index 00000000..0aebb363 --- /dev/null +++ b/src/main/java/com/salesforce/hbase/index/write/recovery/StoreFailuresInCachePolicy.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.salesforce.hbase.index.write.recovery; + +import java.util.List; + +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.regionserver.HRegion; + +import com.google.common.collect.Multimap; +import com.salesforce.hbase.index.exception.MutliIndexWriteFailureException; +import com.salesforce.hbase.index.table.HTableInterfaceReference; +import com.salesforce.hbase.index.write.IndexFailurePolicy; +import com.salesforce.hbase.index.write.KillServerOnFailurePolicy; + +/** + * Tracks any failed writes in The {@link PerRegionIndexWriteCache}, given a + * {@link MutliIndexWriteFailureException} (which is thrown from the + * {@link TrackingParallelWriterIndexCommitter}. Any other exception failure causes the a server + * abort via the usual {@link KillServerOnFailurePolicy}. + */ +public class StoreFailuresInCachePolicy implements IndexFailurePolicy { + + private KillServerOnFailurePolicy delegate; + private PerRegionIndexWriteCache cache; + private HRegion region; + + /** + * @param failedIndexEdits cache to update when we find a failure + */ + public StoreFailuresInCachePolicy(PerRegionIndexWriteCache failedIndexEdits) { + this.cache = failedIndexEdits; + } + + @Override + public void setup(Stoppable parent, RegionCoprocessorEnvironment env) { + this.region = env.getRegion(); + this.delegate = new KillServerOnFailurePolicy(); + this.delegate.setup(parent, env); + + } + + @Override + public void handleFailure(Multimap attempted, Exception cause) { + // if its not an exception we can handle, let the delegate take care of it + if (!(cause instanceof MutliIndexWriteFailureException)) { + delegate.handleFailure(attempted, cause); + } + List failedTables = + ((MutliIndexWriteFailureException) cause).getFailedTables(); + for (HTableInterfaceReference table : failedTables) { + cache.addEdits(this.region, table, attempted.get(table)); + } + } + + + @Override + public void stop(String why) { + this.delegate.stop(why); + } + + @Override + public boolean isStopped() { + return this.delegate.isStopped(); + } +} \ No newline at end of file diff --git a/src/main/java/com/salesforce/hbase/index/write/recovery/TrackingParallelWriterIndexCommitter.java b/src/main/java/com/salesforce/hbase/index/write/recovery/TrackingParallelWriterIndexCommitter.java new file mode 100644 index 00000000..2dd03d4b --- /dev/null +++ b/src/main/java/com/salesforce/hbase/index/write/recovery/TrackingParallelWriterIndexCommitter.java @@ -0,0 +1,266 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index.write.recovery; + +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; + +import com.google.common.collect.Multimap; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.salesforce.hbase.index.CapturingAbortable; +import com.salesforce.hbase.index.exception.CannotReachSingleIndexException; +import com.salesforce.hbase.index.exception.MutliIndexWriteFailureException; +import com.salesforce.hbase.index.table.CachingHTableFactory; +import com.salesforce.hbase.index.table.HTableFactory; +import com.salesforce.hbase.index.table.HTableInterfaceReference; +import com.salesforce.hbase.index.write.IndexCommitter; +import com.salesforce.hbase.index.write.IndexWriter; +import com.salesforce.hbase.index.write.ParallelWriterIndexCommitter; + +/** + * Like the {@link ParallelWriterIndexCommitter}, but blocks until all writes have attempted to + * allow the caller to retrieve the failed and succeeded index updates. Therefore, this class will + * be a lot slower, in the face of failures, when compared to the + * {@link ParallelWriterIndexCommitter} (though as fast for writes), so it should be used only when + * you need to at least attempt all writes and know their result; for instance, this is fine for + * doing WAL recovery - its not a performance intensive situation and we want to limit the the edits + * we need to retry. + *

    + * On failure to {@link #write(Multimap)}, we return a {@link MutliIndexWriteFailureException} that + * contains the list of {@link HTableInterfaceReference} that didn't complete successfully. + *

    + * Failures to write to the index can happen several different ways: + *

      + *
    1. this is {@link #stop(String) stopped} or aborted (via the passed {@link Abortable}. + * This causing any pending tasks to fail whatever they are doing as fast as possible. Any writes + * that have not begun are not even attempted and marked as failures.
      2. + *
    2. A batch write fails. This is the generic HBase write failure - it may occur because the index + * table is not available, .META. or -ROOT- is unavailable, or any other (of many) possible HBase + * exceptions.
      3. + *
    + * Regardless of how the write fails, we still wait for all writes to complete before passing the + * failure back to the client. + */ +public class TrackingParallelWriterIndexCommitter implements IndexCommitter { + private static final Log LOG = LogFactory.getLog(TrackingParallelWriterIndexCommitter.class); + + private ListeningExecutorService writerPool; + private HTableFactory factory; + private CapturingAbortable abortable; + private Stoppable stopped; + + @Override + public void setup(IndexWriter parent, RegionCoprocessorEnvironment env) { + Configuration conf = env.getConfiguration(); + setup(ParallelWriterIndexCommitter.getDefaultDelegateHTableFactory(env), + ParallelWriterIndexCommitter.getDefaultExecutor(conf), + env.getRegionServerServices(), parent, CachingHTableFactory.getCacheSize(conf)); + } + + /** + * Setup this. + *

    + * Exposed for TESTING + */ + void setup(HTableFactory factory, ExecutorService pool, Abortable abortable, Stoppable stop, + int cacheSize) { + this.writerPool = MoreExecutors.listeningDecorator(pool); + this.factory = new CachingHTableFactory(factory, cacheSize); + this.abortable = new CapturingAbortable(abortable); + this.stopped = stop; + } + + @Override + public void write(Multimap toWrite) + throws MutliIndexWriteFailureException { + // track the failures. We synchronize access to the failures via the basics synchronization + // mechanisms. Its faster than using a fully concurrent list since we only need to synchronize + // on add. This list is never exposed until all tasks have completed, at which point it is + // read-only. + final List failures = new LinkedList(); + Set>> entries = toWrite.asMap().entrySet(); + CompletionService ops = new ExecutorCompletionService(this.writerPool); + + // we can use a simple counter here because its ever only modified by the waiting thread + int completedWrites = 0; + + // go through each entry, attempt to write it + for (Entry> entry : entries) { + final HTableInterfaceReference tableReference = entry.getKey(); + // early exit - no need to submit new tasks if we are shutting down + if (this.stopped.isStopped() || this.abortable.isAborted()) { + // we continue to add all the remaining tables that we didn't write because of the external + // quit notification (stop/abort) + synchronized (failures) { + failures.add(tableReference); + } + // this write is considered complete, even without enquing the task + completedWrites++; + continue; + } + + // get the mutations for each table. We leak the implementation here a little bit to save + // doing a complete copy over of all the index update for each table. + final List mutations = (List) entry.getValue(); + + /* + * Write a batch of index updates to an index table. This operation stops (is cancelable) via + * two mechanisms: (1) setting aborted or stopped on the IndexWriter or, (2) interrupting the + * running thread. The former will only work if we are not in the midst of writing the current + * batch to the table, though we do check these status variables before starting and before + * writing the batch. The latter usage, interrupting the thread, will work in the previous + * situations as was at some points while writing the batch, depending on the underlying + * writer implementation (HTableInterface#batch is blocking, but doesn't elaborate when is + * supports an interrupt). + */ + ops.submit(new Callable() { + + /** + * Do the actual write to the primary table. We don't need to worry about closing the table + * because that is handled the {@link CachingHTableFactory}. + */ + @Override + public Void call() throws Exception { + try { + // this may have been queued, but there was an abort/stop so we try to early exit + throwFailureIfDone(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Writing index update:" + mutations + " to table: " + tableReference); + } + HTableInterface table = factory.getTable(tableReference.get()); + throwFailureIfDone(); + table.batch(mutations); + } catch (InterruptedException e) { + addToFailures(tableReference); + // reset the interrupt status on the thread + Thread.currentThread().interrupt(); + throw e; + } catch (Exception e) { + addToFailures(tableReference); + throw e; + } + return null; + } + + private void throwFailureIfDone() throws CannotReachSingleIndexException { + if (stopped.isStopped() || abortable.isAborted() + || Thread.currentThread().isInterrupted()) { + throw new CannotReachSingleIndexException( + "Pool closed, not attempting to write to the index!", null); + } + + } + + private void addToFailures(HTableInterfaceReference table) { + synchronized (failures) { + failures.add(tableReference); + } + } + }); + } + + boolean interrupted = false; + + /* + * wait for all index writes to complete. This is mainly where we differ from the + * ParallelWriterIndexCommiter - we wait for all writes to complete, even if there is a failure, + * to ensure that we try to write as many tables as possible. + */ + while (completedWrites < entries.size()) { + Future status = null; + try { + status = ops.take(); + } catch (InterruptedException e) { + LOG.info("Index writer interrupted, continuing if not aborted or stopped."); + // reset the interrupt status so we can wait out that latch + interrupted = true; + continue; + } + try { + // we don't care what the status is - success is binary, so no error == success + status.get(); + } catch (CancellationException e) { + // if we get a cancellation, we already failed for some other reason, so we can ignore it + LOG.debug("Found canceled index write - ignoring!"); + } catch (ExecutionException e) { + LOG.error("Found a failed index update!", e.getCause()); + } catch (InterruptedException e) { + assert false : "Should never be interrupted getting a completed future."; + LOG.error("Got a completed future, but interrupted while getting the result"); + } + // no matter what happens, as long as we get a status, we update the completed writes + completedWrites++; + } + + // reset the interrupt status after we are done + if (interrupted) { + Thread.currentThread().interrupt(); + } + + // if any of the tasks failed, then we need to propagate the failure + if (failures.size() > 0) { + // make the list unmodifiable to avoid any more synchronization concerns + throw new MutliIndexWriteFailureException(Collections.unmodifiableList(failures)); + } + return; + } + + @Override + public void stop(String why) { + LOG.info("Shutting down " + this.getClass().getSimpleName()); + this.writerPool.shutdownNow(); + this.factory.shutdown(); + } + + @Override + public boolean isStopped() { + return this.stopped.isStopped(); + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/IndexedHLogReader.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/IndexedHLogReader.java index 18528739..904b44c0 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/IndexedHLogReader.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/IndexedHLogReader.java @@ -21,6 +21,13 @@ * This is a little bit of a painful way of going about this, but saves the effort of hacking the * HBase source (and deal with getting it reviewed and backported, etc.) and still works. */ +/* + * TODO: Support splitting index updates into their own WAL entries on recovery (basically, just + * queue them up in next), if we know that the region was on the server when it crashed. However, + * this is kind of difficult as we need to know a lot of things the state of the system - basically, + * we need to track which of the regions were on the server when it crashed only only split those + * edits out into their respective regions. + */ public class IndexedHLogReader implements Reader { private SequenceFileLogReader delegate; @@ -40,8 +47,8 @@ private static class IndexedWALReader extends SequenceFileLogReader.WALReader { /** * we basically have to reproduce what the SequenceFile.Reader is doing in next(), but without - * the check out the value class, since we have a special value class that doesn't directly - * match what was specified in the file header + * the check on the value class, since we have a special value class that doesn't directly match + * what was specified in the file header */ @Override public synchronized boolean next(Writable key, Writable val) throws IOException { diff --git a/src/test/java/com/salesforce/hbase/index/IndexTestingUtils.java b/src/test/java/com/salesforce/hbase/index/IndexTestingUtils.java index 41499302..ba569920 100644 --- a/src/test/java/com/salesforce/hbase/index/IndexTestingUtils.java +++ b/src/test/java/com/salesforce/hbase/index/IndexTestingUtils.java @@ -42,8 +42,13 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.wal.IndexedHLogReader; +import org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec; +import org.apache.hadoop.hbase.regionserver.wal.WALEditCodec; import org.apache.hadoop.hbase.util.Bytes; +import com.salesforce.hbase.index.util.IndexManagementUtil; + /** * Utility class for testing indexing @@ -61,6 +66,9 @@ private IndexTestingUtils() { public static void setupConfig(Configuration conf) { conf.setInt(MASTER_INFO_PORT_KEY, -1); conf.setInt(RS_INFO_PORT_KEY, -1); + // setup our codec and reader, so we get proper replay/write + conf.set(IndexManagementUtil.HLOG_READER_IMPL_KEY, IndexedHLogReader.class.getName()); + conf.set(WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY, IndexedWALEditCodec.class.getName()); } /** * Verify the state of the index table between the given key and time ranges against the list of diff --git a/src/test/java/com/salesforce/hbase/index/write/TestIndexWriter.java b/src/test/java/com/salesforce/hbase/index/write/TestIndexWriter.java index f966e01b..e1415fb2 100644 --- a/src/test/java/com/salesforce/hbase/index/write/TestIndexWriter.java +++ b/src/test/java/com/salesforce/hbase/index/write/TestIndexWriter.java @@ -60,9 +60,10 @@ import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import com.salesforce.hbase.index.CannotReachIndexException; import com.salesforce.hbase.index.StubAbortable; import com.salesforce.hbase.index.TableName; +import com.salesforce.hbase.index.exception.CannotReachSingleIndexException; +import com.salesforce.hbase.index.exception.IndexWriteException; import com.salesforce.hbase.index.util.ImmutableBytesPtr; public class TestIndexWriter { @@ -206,7 +207,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable { try { writer.write(indexUpdates); fail("Should not have successfully completed all index writes"); - } catch (CannotReachIndexException e) { + } catch (CannotReachSingleIndexException e) { LOG.info("Correctly got a failure to reach the index", e); // should have correctly gotten the correct abort, so let the next task execute waitOnAbortedLatch.countDown(); @@ -271,7 +272,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable { public void run() { try { writer.write(indexUpdates); - } catch (CannotReachIndexException e) { + } catch (IndexWriteException e) { failedWrite[0] = true; } } diff --git a/src/test/java/com/salesforce/hbase/index/write/TestWALRecoveryCaching.java b/src/test/java/com/salesforce/hbase/index/write/TestWALRecoveryCaching.java new file mode 100644 index 00000000..16abcacb --- /dev/null +++ b/src/test/java/com/salesforce/hbase/index/write/TestWALRecoveryCaching.java @@ -0,0 +1,375 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index.write; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.wal.HLogKey; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; + +import com.google.common.collect.Multimap; +import com.salesforce.hbase.index.IndexTestingUtils; +import com.salesforce.hbase.index.Indexer; +import com.salesforce.hbase.index.TableName; +import com.salesforce.hbase.index.covered.example.ColumnGroup; +import com.salesforce.hbase.index.covered.example.CoveredColumn; +import com.salesforce.hbase.index.covered.example.CoveredColumnIndexSpecifierBuilder; +import com.salesforce.hbase.index.covered.example.CoveredColumnIndexer; +import com.salesforce.hbase.index.table.HTableInterfaceReference; +import com.salesforce.hbase.index.util.IndexManagementUtil; +import com.salesforce.hbase.index.write.recovery.PerRegionIndexWriteCache; +import com.salesforce.hbase.index.write.recovery.StoreFailuresInCachePolicy; + +/** + * When a regionserver crashes, its WAL is split and then replayed to the server. If the index + * region was present on the same server, we have to make a best effort to not kill the server for + * not succeeding on index writes while the index region is coming up. + */ +public class TestWALRecoveryCaching { + + private static final Log LOG = LogFactory.getLog(TestWALRecoveryCaching.class); + private static final long ONE_SEC = 1000; + private static final long ONE_MIN = 60 * ONE_SEC; + private static final long TIMEOUT = ONE_MIN; + + @Rule + public TableName testTable = new TableName(); + + private String getIndexTableName() { + return this.testTable.getTableNameString() + "_index"; + } + + // ----------------------------------------------------------------------------------------------- + // Warning! The classes here rely on this static. Adding multiple tests to this class and running + // them concurrently could have unexpected results (including, but not limited to, odd failures + // and flapping tests). + // ----------------------------------------------------------------------------------------------- + private static CountDownLatch allowIndexTableToRecover; + + public static class IndexTableBlockingReplayObserver extends BaseRegionObserver { + + @Override + public void preWALRestore(ObserverContext env, HRegionInfo info, + HLogKey logKey, WALEdit logEdit) throws IOException { + try { + LOG.debug("Restoring logs for index table"); + if (allowIndexTableToRecover != null) { + allowIndexTableToRecover.await(); + LOG.debug("Completed index table recovery wait latch"); + } + } catch (InterruptedException e) { + Assert.fail("Should not be interrupted while waiting to allow the index to restore WALs."); + } + } + } + + public static class ReleaseLatchOnFailurePolicy extends StoreFailuresInCachePolicy { + + /** + * @param failedIndexEdits + */ + public ReleaseLatchOnFailurePolicy(PerRegionIndexWriteCache failedIndexEdits) { + super(failedIndexEdits); + } + + @Override + public void handleFailure(Multimap attempted, + Exception cause) { + LOG.debug("Found index update failure!"); + if (allowIndexTableToRecover != null) { + LOG.info("failed index write on WAL recovery - allowing index table to be restored."); + allowIndexTableToRecover.countDown(); + } + super.handleFailure(attempted, cause); + } + + } + + @Test + public void testWaitsOnIndexRegionToReload() throws Exception { + HBaseTestingUtility util = new HBaseTestingUtility(); + Configuration conf = util.getConfiguration(); + + // setup other useful stats + IndexTestingUtils.setupConfig(conf); + conf.setBoolean(Indexer.CHECK_VERSION_CONF_KEY, false); + + // make sure everything is setup correctly + IndexManagementUtil.ensureMutableIndexingCorrectlyConfigured(conf); + + // start the cluster with 2 rs + util.startMiniCluster(2); + + HBaseAdmin admin = util.getHBaseAdmin(); + // setup the index + byte[] family = Bytes.toBytes("family"); + byte[] qual = Bytes.toBytes("qualifier"); + byte[] nonIndexedFamily = Bytes.toBytes("nonIndexedFamily"); + String indexedTableName = getIndexTableName(); + ColumnGroup columns = new ColumnGroup(indexedTableName); + columns.add(new CoveredColumn(family, qual)); + CoveredColumnIndexSpecifierBuilder builder = new CoveredColumnIndexSpecifierBuilder(); + builder.addIndexGroup(columns); + + // create the primary table w/ indexing enabled + HTableDescriptor primaryTable = new HTableDescriptor(testTable.getTableName()); + primaryTable.addFamily(new HColumnDescriptor(family)); + primaryTable.addFamily(new HColumnDescriptor(nonIndexedFamily)); + builder.addArbitraryConfigForTesting(Indexer.RecoveryFailurePolicyKeyForTesting, + ReleaseLatchOnFailurePolicy.class.getName()); + builder.build(primaryTable); + admin.createTable(primaryTable); + + // create the index table + HTableDescriptor indexTableDesc = new HTableDescriptor(Bytes.toBytes(getIndexTableName())); + indexTableDesc.addCoprocessor(IndexTableBlockingReplayObserver.class.getName()); + CoveredColumnIndexer.createIndexTable(admin, indexTableDesc); + + // figure out where our tables live + ServerName shared = + ensureTablesLiveOnSameServer(util.getMiniHBaseCluster(), Bytes.toBytes(indexedTableName), + testTable.getTableName()); + + // load some data into the table + Put p = new Put(Bytes.toBytes("row")); + p.add(family, qual, Bytes.toBytes("value")); + HTable primary = new HTable(conf, testTable.getTableName()); + primary.put(p); + primary.flushCommits(); + + // turn on the recovery latch + allowIndexTableToRecover = new CountDownLatch(1); + + // kill the server where the tables live - this should trigger distributed log splitting + // find the regionserver that matches the passed server + List online = new ArrayList(); + online.addAll(getRegionsFromServerForTable(util.getMiniHBaseCluster(), shared, + testTable.getTableName())); + online.addAll(getRegionsFromServerForTable(util.getMiniHBaseCluster(), shared, + Bytes.toBytes(indexedTableName))); + + // log all the current state of the server + LOG.info("Current Server/Region paring: "); + for (RegionServerThread t : util.getMiniHBaseCluster().getRegionServerThreads()) { + // check all the conditions for the server to be done + HRegionServer server = t.getRegionServer(); + if (server.isStopping() || server.isStopped() || server.isAborted()) { + LOG.info("\t== Offline: " + server.getServerName()); + continue; + } + List regions = server.getOnlineRegions(); + LOG.info("\t" + server.getServerName() + " regions: " + regions); + } + + LOG.debug("Killing server " + shared); + util.getMiniHBaseCluster().killRegionServer(shared); + LOG.debug("Waiting on server " + shared + "to die"); + util.getMiniHBaseCluster().waitForRegionServerToStop(shared, TIMEOUT); + // force reassign the regions from the table + // LOG.debug("Forcing region reassignment from the killed server: " + shared); + // for (HRegion region : online) { + // util.getMiniHBaseCluster().getMaster().assign(region.getRegionName()); + // } + System.out.println(" ====== Killed shared server ==== "); + + // make a second put that (1), isn't indexed, so we can be sure of the index state and (2) + // ensures that our table is back up + Put p2 = new Put(p.getRow()); + p2.add(nonIndexedFamily, Bytes.toBytes("Not indexed"), Bytes.toBytes("non-indexed value")); + primary.put(p2); + primary.flushCommits(); + + // make sure that we actually failed the write once (within a 5 minute window) + assertTrue("Didn't find an error writing to index table within timeout!", + allowIndexTableToRecover.await(ONE_MIN * 5, TimeUnit.MILLISECONDS)); + + // scan the index to make sure it has the one entry, (that had to be replayed from the WAL, + // since we hard killed the server) + Scan s = new Scan(); + HTable index = new HTable(conf, getIndexTableName()); + ResultScanner scanner = index.getScanner(s); + int count = 0; + for (Result r : scanner) { + LOG.info("Got index table result:" + r); + count++; + } + assertEquals("Got an unexpected found of index rows", 1, count); + + // cleanup + scanner.close(); + index.close(); + primary.close(); + util.shutdownMiniCluster(); + } + + /** + * @param miniHBaseCluster + * @param server + * @param bs + * @return + */ + private List getRegionsFromServerForTable(MiniHBaseCluster cluster, ServerName server, + byte[] table) { + List online = Collections.emptyList(); + for (RegionServerThread rst : cluster.getRegionServerThreads()) { + // if its the server we are going to kill, get the regions we want to reassign + if (rst.getRegionServer().getServerName().equals(server)) { + online = rst.getRegionServer().getOnlineRegions(table); + break; + } + } + return online; + } + + /** + * @param miniHBaseCluster + * @param indexedTableName + * @param tableNameString + */ + private ServerName ensureTablesLiveOnSameServer(MiniHBaseCluster cluster, byte[] indexTable, + byte[] primaryTable) throws Exception { + + ServerName shared = getSharedServer(cluster, indexTable, primaryTable); + boolean tryIndex = true; + while (shared == null) { + + // start killing servers until we get an overlap + Set servers; + byte[] table = null; + // switch which server we kill each time to get region movement + if (tryIndex) { + table = indexTable; + } else { + table = primaryTable; + } + servers = getServersForTable(cluster, table); + tryIndex = !tryIndex; + for (ServerName server : servers) { + // find the regionserver that matches the passed server + List online = getRegionsFromServerForTable(cluster, server, table); + + LOG.info("Shutting down and reassigning regions from " + server); + cluster.stopRegionServer(server); + cluster.waitForRegionServerToStop(server, TIMEOUT); + + // force reassign the regions from the table + for (HRegion region : online) { + cluster.getMaster().assign(region.getRegionName()); + } + + LOG.info("Starting region server:" + server.getHostname()); + cluster.startRegionServer(server.getHostname()); + + cluster.waitForRegionServerToStart(server.getHostname(), TIMEOUT); + + // start a server to get back to the base number of servers + LOG.info("STarting server to replace " + server); + cluster.startRegionServer(); + break; + } + + shared = getSharedServer(cluster, indexTable, primaryTable); + } + return shared; + } + + /** + * @param cluster + * @param indexTable + * @param primaryTable + * @return + * @throws Exception + */ + private ServerName getSharedServer(MiniHBaseCluster cluster, byte[] indexTable, + byte[] primaryTable) throws Exception { + Set indexServers = getServersForTable(cluster, indexTable); + Set primaryServers = getServersForTable(cluster, primaryTable); + + Set joinSet = new HashSet(indexServers); + joinSet.addAll(primaryServers); + // if there is already an overlap, then find it and return it + if (joinSet.size() < indexServers.size() + primaryServers.size()) { + // find the first overlapping server + for (ServerName server : joinSet) { + if (indexServers.contains(server) && primaryServers.contains(server)) { + return server; + } + } + throw new RuntimeException( + "Couldn't find a matching server on which both the primary and index table live, " + + "even though they have overlapping server sets"); + } + return null; + } + + private Set getServersForTable(MiniHBaseCluster cluster, byte[] table) + throws Exception { + List indexRegions = cluster.getRegions(table); + Set indexServers = new HashSet(); + for (HRegion region : indexRegions) { + indexServers.add(cluster.getServerHoldingRegion(region.getRegionName())); + } + return indexServers; + } +} \ No newline at end of file diff --git a/src/test/java/com/salesforce/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java b/src/test/java/com/salesforce/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java new file mode 100644 index 00000000..9f398dab --- /dev/null +++ b/src/test/java/com/salesforce/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java @@ -0,0 +1,178 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index.write.recovery; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map.Entry; +import java.util.Set; + +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; + +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import com.salesforce.hbase.index.table.HTableInterfaceReference; +import com.salesforce.hbase.index.util.ImmutableBytesPtr; +import com.salesforce.hbase.index.write.recovery.PerRegionIndexWriteCache; + +public class TestPerRegionIndexWriteCache { + + private static final byte[] row = Bytes.toBytes("row"); + private static final byte[] family = Bytes.toBytes("family"); + private static final byte[] qual = Bytes.toBytes("qual"); + private static final byte[] val = Bytes.toBytes("val"); + + Put p = new Put(row); + Put p2 = new Put(Bytes.toBytes("other row")); + { + p.add(family, qual, val); + p2.add(family, qual, val); + } + + + HRegion r1 = new HRegion() { + @Override + public int hashCode() { + return 1; + } + + @Override + public String toString() { + return "testRegion1"; + } + }; + HRegion r2 = new HRegion() { + @Override + public int hashCode() { + return 2; + } + + @Override + public String toString() { + return "testRegion1"; + } + }; + + @Test + public void testAddRemoveSingleRegion() { + PerRegionIndexWriteCache cache = new PerRegionIndexWriteCache(); + HTableInterfaceReference t1 = new HTableInterfaceReference(new ImmutableBytesPtr(Bytes.toBytes("t1"))); + List mutations = new ArrayList(); + mutations.add(p); + cache.addEdits(r1, t1, mutations); + Multimap edits = cache.getEdits(r1); + Set>> entries = edits.asMap().entrySet(); + assertEquals("Got more than one table in the the edit map!", 1, entries.size()); + for (Entry> entry : entries) { + //ensure that we are still storing a list here - otherwise it breaks the parallel writer implementation + final List stored = (List) entry.getValue(); + assertEquals("Got an unexpected amount of mutations in the entry", 1, stored.size()); + assertEquals("Got an unexpected mutation in the entry", p, stored.get(0)); + } + + // ensure that a second get doesn't have any more edits. This ensures that we don't keep + // references around to these edits and have a memory leak + assertNull("Got an entry for a region we removed", cache.getEdits(r1)); + } + + @Test + public void testMultipleAddsForSingleRegion() { + PerRegionIndexWriteCache cache = new PerRegionIndexWriteCache(); + HTableInterfaceReference t1 = + new HTableInterfaceReference(new ImmutableBytesPtr(Bytes.toBytes("t1"))); + List mutations = Lists. newArrayList(p); + cache.addEdits(r1, t1, mutations); + + // add a second set + mutations = Lists. newArrayList(p2); + cache.addEdits(r1, t1, mutations); + + Multimap edits = cache.getEdits(r1); + Set>> entries = edits.asMap().entrySet(); + assertEquals("Got more than one table in the the edit map!", 1, entries.size()); + for (Entry> entry : entries) { + // ensure that we are still storing a list here - otherwise it breaks the parallel writer + // implementation + final List stored = (List) entry.getValue(); + assertEquals("Got an unexpected amount of mutations in the entry", 2, stored.size()); + assertEquals("Got an unexpected mutation in the entry", p, stored.get(0)); + assertEquals("Got an unexpected mutation in the entry", p2, stored.get(1)); + } + } + + @Test + public void testMultipleRegions() { + PerRegionIndexWriteCache cache = new PerRegionIndexWriteCache(); + HTableInterfaceReference t1 = + new HTableInterfaceReference(new ImmutableBytesPtr(Bytes.toBytes("t1"))); + List mutations = Lists. newArrayList(p); + List m2 = Lists. newArrayList(p2); + // add each region + cache.addEdits(r1, t1, mutations); + cache.addEdits(r2, t1, m2); + + // check region1 + Multimap edits = cache.getEdits(r1); + Set>> entries = edits.asMap().entrySet(); + assertEquals("Got more than one table in the the edit map!", 1, entries.size()); + for (Entry> entry : entries) { + // ensure that we are still storing a list here - otherwise it breaks the parallel writer + // implementation + final List stored = (List) entry.getValue(); + assertEquals("Got an unexpected amount of mutations in the entry for region1", 1, + stored.size()); + assertEquals("Got an unexpected mutation in the entry for region2", p, stored.get(0)); + } + + // check region2 + edits = cache.getEdits(r2); + entries = edits.asMap().entrySet(); + assertEquals("Got more than one table in the the edit map!", 1, entries.size()); + for (Entry> entry : entries) { + // ensure that we are still storing a list here - otherwise it breaks the parallel writer + // implementation + final List stored = (List) entry.getValue(); + assertEquals("Got an unexpected amount of mutations in the entry for region2", 1, + stored.size()); + assertEquals("Got an unexpected mutation in the entry for region2", p2, stored.get(0)); + } + + + // ensure that a second get doesn't have any more edits. This ensures that we don't keep + // references around to these edits and have a memory leak + assertNull("Got an entry for a region we removed", cache.getEdits(r1)); + } +} \ No newline at end of file From e712600922df515cfa2d3cb4e3858d744673cdc5 Mon Sep 17 00:00:00 2001 From: maryannxue Date: Mon, 30 Sep 2013 21:50:32 -0400 Subject: [PATCH 089/102] bug fix --- .../phoenix/compile/JoinCompiler.java | 29 ++++++++++++++----- .../phoenix/compile/QueryCompiler.java | 16 +++++----- .../coprocessor/HashJoinRegionScanner.java | 6 +++- .../phoenix/execute/HashJoinPlan.java | 10 +++++-- .../query/ConnectionQueryServicesImpl.java | 3 ++ .../phoenix/util/ImmutableBytesPtr.java | 8 +++++ 6 files changed, 51 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java index 7e82bd71..aa19407c 100644 --- a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java @@ -97,9 +97,12 @@ private JoinSpec(SelectStatement statement, ColumnResolver resolver) throws SQLE this.mainTable = tableRefIter.next(); this.select = extractFromSelect(selectList, mainTable, resolver); this.joinTables = new ArrayList(tableNodes.size() - 1); + this.preFilters = new ArrayList(); + this.postFilters = new ArrayList(); ColumnParseNodeVisitor visitor = new ColumnParseNodeVisitor(resolver); TableNode tableNode = null; - for (; iter.hasNext(); tableNode = iter.next()) { + while (iter.hasNext()) { + tableNode = iter.next(); if (!(tableNode instanceof JoinTableNode)) throw new SQLFeatureNotSupportedException("Full joins not supported."); JoinTableNode joinTableNode = (JoinTableNode) tableNode; @@ -107,17 +110,27 @@ private JoinSpec(SelectStatement statement, ColumnResolver resolver) throws SQLE joinTables.add(joinTable); joinTableNode.getOnNode().accept(visitor); } - statement.getWhere().accept(new WhereNodeVisitor(resolver)); + if (statement.getWhere() != null) { + statement.getWhere().accept(new WhereNodeVisitor(resolver)); + } for (AliasedNode node : selectList) { node.getNode().accept(visitor); } - statement.getWhere().accept(visitor); - for (ParseNode node : statement.getGroupBy()) { - node.accept(visitor); + if (statement.getWhere() != null) { + statement.getWhere().accept(visitor); } - statement.getHaving().accept(visitor); - for (OrderByNode node : statement.getOrderBy()) { - node.getNode().accept(visitor); + if (statement.getGroupBy() != null) { + for (ParseNode node : statement.getGroupBy()) { + node.accept(visitor); + } + } + if (statement.getHaving() != null) { + statement.getHaving().accept(visitor); + } + if (statement.getOrderBy() != null) { + for (OrderByNode node : statement.getOrderBy()) { + node.getNode().accept(visitor); + } } this.columnRefs = visitor.getColumnRefMap().keySet(); } diff --git a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java index cbfd7805..009bbf13 100644 --- a/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/QueryCompiler.java @@ -140,6 +140,7 @@ protected QueryPlan compile(SelectStatement statement, List binds, Scan @SuppressWarnings("unchecked") protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement statement, List binds, JoinSpec join) throws SQLException { + byte[] emptyByteArray = new byte[0]; List joinTables = join.getJoinTables(); if (joinTables.isEmpty()) { context.setCurrentTable(join.getMainTable()); @@ -159,7 +160,7 @@ protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement s QueryPlan[] joinPlans = new QueryPlan[count]; for (int i = 0; i < count; i++) { JoinTable joinTable = joinTables.get(i); - joinIds[i] = new ImmutableBytesPtr(); // place-holder + joinIds[i] = new ImmutableBytesPtr(emptyByteArray); // place-holder joinExpressions[i] = joinTable.compileLeftTableConditions(context); hashExpressions[i] = joinTable.compileRightTableConditions(context); joinTypes[i] = joinTable.getType(); @@ -174,10 +175,9 @@ protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement s } Expression postJoinFilterExpression = JoinCompiler.compilePostJoinFilterExpression(context, join, null); HashJoinInfo joinInfo = new HashJoinInfo(joinIds, joinExpressions, joinTypes, postJoinFilterExpression); - HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); ScanProjector.serializeProjectorIntoScan(context.getScan(), join.getScanProjector()); BasicQueryPlan plan = compileSingleQuery(context, JoinCompiler.getSubqueryWithoutJoin(statement, join), binds); - return new HashJoinPlan(plan, joinIds, hashExpressions, joinPlans); + return new HashJoinPlan(plan, joinInfo, hashExpressions, joinPlans); } JoinTable lastJoinTable = joinTables.get(joinTables.size() - 1); @@ -199,15 +199,14 @@ protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement s } StatementContext lhsCtx = new StatementContext(statement, connection, context.getResolver(), binds, subScan, true, context.getHashClient()); QueryPlan lhsPlan = compileJoinQuery(lhsCtx, lhs, binds, lhsJoin); - ImmutableBytesPtr[] joinIds = new ImmutableBytesPtr[] {new ImmutableBytesPtr()}; + ImmutableBytesPtr[] joinIds = new ImmutableBytesPtr[] {new ImmutableBytesPtr(emptyByteArray)}; Expression postJoinFilterExpression = JoinCompiler.compilePostJoinFilterExpression(context, join, lastJoinTable); List joinExpressions = lastJoinTable.compileRightTableConditions(context); List hashExpressions = lastJoinTable.compileLeftTableConditions(context); HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {joinExpressions}, new JoinType[] {JoinType.Left}, postJoinFilterExpression); - HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); ScanProjector.serializeProjectorIntoScan(context.getScan(), lastJoinTable.getScanProjector()); BasicQueryPlan rhsPlan = compileSingleQuery(context, rhs, binds); - return new HashJoinPlan(rhsPlan, joinIds, new List[] {hashExpressions}, new QueryPlan[] {lhsPlan}); + return new HashJoinPlan(rhsPlan, joinInfo, new List[] {hashExpressions}, new QueryPlan[] {lhsPlan}); } SelectStatement lhs = JoinCompiler.getSubQueryWithoutLastJoinAsFinalPlan(statement, join); @@ -222,15 +221,14 @@ protected QueryPlan compileJoinQuery(StatementContext context, SelectStatement s join.projectColumns(subScan, lastJoinTable.getTable()); ScanProjector.serializeProjectorIntoScan(subScan, lastJoinTable.getScanProjector()); QueryPlan rhsPlan = compile(rhs, binds, subScan); - ImmutableBytesPtr[] joinIds = new ImmutableBytesPtr[] {new ImmutableBytesPtr()}; + ImmutableBytesPtr[] joinIds = new ImmutableBytesPtr[] {new ImmutableBytesPtr(emptyByteArray)}; Expression postJoinFilterExpression = JoinCompiler.compilePostJoinFilterExpression(context, join, null); List joinExpressions = lastJoinTable.compileLeftTableConditions(context); List hashExpressions = lastJoinTable.compileRightTableConditions(context); HashJoinInfo joinInfo = new HashJoinInfo(joinIds, new List[] {joinExpressions}, new JoinType[] {JoinType.Left}, postJoinFilterExpression); - HashJoinInfo.serializeHashJoinIntoScan(context.getScan(), joinInfo); ScanProjector.serializeProjectorIntoScan(context.getScan(), join.getScanProjector()); BasicQueryPlan lhsPlan = compileSingleQuery(context, lhs, binds); - return new HashJoinPlan(lhsPlan, joinIds, new List[] {hashExpressions}, new QueryPlan[] {rhsPlan}); + return new HashJoinPlan(lhsPlan, joinInfo, new List[] {hashExpressions}, new QueryPlan[] {rhsPlan}); } protected BasicQueryPlan compileSingleQuery(StatementContext context, SelectStatement statement, List binds) throws SQLException{ diff --git a/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java b/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java index e2d034c4..ce785d33 100644 --- a/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java +++ b/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.util.Bytes; import com.salesforce.phoenix.cache.*; import com.salesforce.phoenix.expression.Expression; @@ -101,7 +102,10 @@ private void processResults(List result, boolean hasLimit) throws IOEx boolean cont = true; for (int i = 0; i < count; i++) { ImmutableBytesPtr key = TupleUtil.getConcatenatedValue(tuple, joinInfo.getJoinExpressions()[i]); - HashCache hashCache = (HashCache)cache.getServerCache(joinInfo.getJoinIds()[i]); + ImmutableBytesPtr joinId = joinInfo.getJoinIds()[i]; + HashCache hashCache = (HashCache)cache.getServerCache(joinId); + if (hashCache == null) + throw new IOException("Could not find hash cache for joinId: " + Bytes.toString(joinId.get())); tuples[i] = hashCache.get(key); JoinType type = joinInfo.getJoinTypes()[i]; if (type == JoinType.Inner && (tuples[i] == null || tuples[i].isEmpty())) { diff --git a/src/main/java/com/salesforce/phoenix/execute/HashJoinPlan.java b/src/main/java/com/salesforce/phoenix/execute/HashJoinPlan.java index 6bebf6b3..73fd1e8c 100644 --- a/src/main/java/com/salesforce/phoenix/execute/HashJoinPlan.java +++ b/src/main/java/com/salesforce/phoenix/execute/HashJoinPlan.java @@ -15,6 +15,7 @@ import com.salesforce.phoenix.compile.OrderByCompiler.OrderBy; import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.join.HashCacheClient; +import com.salesforce.phoenix.join.HashJoinInfo; import com.salesforce.phoenix.query.KeyRange; import com.salesforce.phoenix.query.Scanner; import com.salesforce.phoenix.schema.TableRef; @@ -23,14 +24,14 @@ public class HashJoinPlan implements QueryPlan { private BasicQueryPlan plan; - private ImmutableBytesPtr[] joinIds; + private HashJoinInfo joinInfo; private List[] hashExpressions; private QueryPlan[] hashPlans; - public HashJoinPlan(BasicQueryPlan plan, ImmutableBytesPtr[] joinIds, + public HashJoinPlan(BasicQueryPlan plan, HashJoinInfo joinInfo, List[] hashExpressions, QueryPlan[] hashPlans) { this.plan = plan; - this.joinIds = joinIds; + this.joinInfo = joinInfo; this.hashExpressions = hashExpressions; this.hashPlans = hashPlans; } @@ -52,6 +53,7 @@ public RowProjector getProjector() { @Override public Scanner getScanner() throws SQLException { + ImmutableBytesPtr[] joinIds = joinInfo.getJoinIds(); assert (joinIds.length == hashExpressions.length && joinIds.length == hashPlans.length); HashCacheClient hashClient = plan.getContext().getHashClient(); @@ -62,6 +64,8 @@ public Scanner getScanner() throws SQLException { ServerCache cache = hashClient.addHashCache(hashPlans[i].getScanner(), hashExpressions[i], plan.getTableRef(), keyRange); joinIds[i].set(cache.getId()); } + HashJoinInfo.serializeHashJoinIntoScan(scan, joinInfo); + return plan.getScanner(); } diff --git a/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java b/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java index e08dc633..2785b573 100644 --- a/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java +++ b/src/main/java/com/salesforce/phoenix/query/ConnectionQueryServicesImpl.java @@ -452,6 +452,9 @@ private HTableDescriptor generateTableDescriptor(byte[] tableName, HTableDescrip if (!descriptor.hasCoprocessor(HashJoiningRegionObserver.class.getName())) { descriptor.addCoprocessor(HashJoiningRegionObserver.class.getName(), null, 1, null); } + if (!descriptor.hasCoprocessor(ServerCachingEndpointImpl.class.getName())) { + descriptor.addCoprocessor(ServerCachingEndpointImpl.class.getName(), null, 1, null); + } // Setup split policy on Phoenix metadata table to ensure that the key values of a Phoenix table // stay on the same region. if (SchemaUtil.isMetaTable(tableName)) { diff --git a/src/main/java/com/salesforce/phoenix/util/ImmutableBytesPtr.java b/src/main/java/com/salesforce/phoenix/util/ImmutableBytesPtr.java index e8631ee3..105019ea 100644 --- a/src/main/java/com/salesforce/phoenix/util/ImmutableBytesPtr.java +++ b/src/main/java/com/salesforce/phoenix/util/ImmutableBytesPtr.java @@ -27,6 +27,9 @@ ******************************************************************************/ package com.salesforce.phoenix.util; +import java.io.DataInput; +import java.io.IOException; + import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; @@ -91,5 +94,10 @@ public void set(final byte [] b, final int offset, final int length) { hashCode = super.hashCode(); } + @Override + public void readFields(final DataInput in) throws IOException { + super.readFields(in); + hashCode = super.hashCode(); + } } From d390d36f06c01354a26a2ed12e676f54492450e1 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Tue, 1 Oct 2013 10:01:24 -0700 Subject: [PATCH 090/102] Kick the build to see if test is flapping --- build.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.txt b/build.txt index a3c5fed8..20585fb8 100644 --- a/build.txt +++ b/build.txt @@ -1,7 +1,7 @@ # Building Phoenix ================ -Phoenix uses Maven (3.X) to build all its necessary resources. +Phoenix uses Maven (3.X) to build all its necessary resources. ## Building from source ======================= From 57e7ad8ca90c43f2d91be79f4c0127f326635a00 Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Tue, 1 Oct 2013 10:54:36 -0700 Subject: [PATCH 091/102] Nit cleanups from index recovery fix --- ...a => MultiIndexWriteFailureException.java} | 4 ++-- ... => SingleIndexWriteFailureException.java} | 7 ++++--- .../write/ParallelWriterIndexCommitter.java | 19 ++++++++++--------- .../recovery/StoreFailuresInCachePolicy.java | 8 ++++---- .../TrackingParallelWriterIndexCommitter.java | 18 +++++++++--------- .../hbase/index/write/TestIndexWriter.java | 4 ++-- 6 files changed, 31 insertions(+), 29 deletions(-) rename src/main/java/com/salesforce/hbase/index/exception/{MutliIndexWriteFailureException.java => MultiIndexWriteFailureException.java} (94%) rename src/main/java/com/salesforce/hbase/index/exception/{CannotReachSingleIndexException.java => SingleIndexWriteFailureException.java} (91%) diff --git a/src/main/java/com/salesforce/hbase/index/exception/MutliIndexWriteFailureException.java b/src/main/java/com/salesforce/hbase/index/exception/MultiIndexWriteFailureException.java similarity index 94% rename from src/main/java/com/salesforce/hbase/index/exception/MutliIndexWriteFailureException.java rename to src/main/java/com/salesforce/hbase/index/exception/MultiIndexWriteFailureException.java index abeb0a2a..6c57750e 100644 --- a/src/main/java/com/salesforce/hbase/index/exception/MutliIndexWriteFailureException.java +++ b/src/main/java/com/salesforce/hbase/index/exception/MultiIndexWriteFailureException.java @@ -35,14 +35,14 @@ * Indicate a failure to write to multiple index tables. */ @SuppressWarnings("serial") -public class MutliIndexWriteFailureException extends IndexWriteException { +public class MultiIndexWriteFailureException extends IndexWriteException { private List failures; /** * @param failures the tables to which the index write did not succeed */ - public MutliIndexWriteFailureException(List failures) { + public MultiIndexWriteFailureException(List failures) { super("Failed to write to multiple index tables"); this.failures = failures; diff --git a/src/main/java/com/salesforce/hbase/index/exception/CannotReachSingleIndexException.java b/src/main/java/com/salesforce/hbase/index/exception/SingleIndexWriteFailureException.java similarity index 91% rename from src/main/java/com/salesforce/hbase/index/exception/CannotReachSingleIndexException.java rename to src/main/java/com/salesforce/hbase/index/exception/SingleIndexWriteFailureException.java index c6658951..66aefe9f 100644 --- a/src/main/java/com/salesforce/hbase/index/exception/CannotReachSingleIndexException.java +++ b/src/main/java/com/salesforce/hbase/index/exception/SingleIndexWriteFailureException.java @@ -35,7 +35,7 @@ * Exception thrown if we cannot successfully write to an index table. */ @SuppressWarnings("serial") -public class CannotReachSingleIndexException extends IndexWriteException { +public class SingleIndexWriteFailureException extends IndexWriteException { private String table; @@ -44,7 +44,7 @@ public class CannotReachSingleIndexException extends IndexWriteException { * @param msg more description of what happened * @param cause original cause */ - public CannotReachSingleIndexException(String msg, Throwable cause) { + public SingleIndexWriteFailureException(String msg, Throwable cause) { super(msg, cause); } @@ -54,7 +54,8 @@ public CannotReachSingleIndexException(String msg, Throwable cause) { * @param mutations mutations that were attempted * @param cause underlying reason for the failure */ - public CannotReachSingleIndexException(String targetTableName, List mutations, Exception cause) { + public SingleIndexWriteFailureException(String targetTableName, List mutations, + Exception cause) { super("Failed to make index update:\n\t table: " + targetTableName + "\n\t edits: " + mutations + "\n\tcause: " + cause == null ? "UNKNOWN" : cause.getMessage(), cause); this.table = targetTableName; diff --git a/src/main/java/com/salesforce/hbase/index/write/ParallelWriterIndexCommitter.java b/src/main/java/com/salesforce/hbase/index/write/ParallelWriterIndexCommitter.java index 4e502d16..3b50efde 100644 --- a/src/main/java/com/salesforce/hbase/index/write/ParallelWriterIndexCommitter.java +++ b/src/main/java/com/salesforce/hbase/index/write/ParallelWriterIndexCommitter.java @@ -58,7 +58,7 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.salesforce.hbase.index.CapturingAbortable; -import com.salesforce.hbase.index.exception.CannotReachSingleIndexException; +import com.salesforce.hbase.index.exception.SingleIndexWriteFailureException; import com.salesforce.hbase.index.table.CachingHTableFactory; import com.salesforce.hbase.index.table.CoprocessorHTableFactory; import com.salesforce.hbase.index.table.HTableFactory; @@ -148,7 +148,7 @@ public static ThreadPoolExecutor getDefaultExecutor(Configuration conf) { @Override public void write(Multimap toWrite) - throws CannotReachSingleIndexException { + throws SingleIndexWriteFailureException { /* * This bit here is a little odd, so let's explain what's going on. Basically, we want to do the * writes in parallel to each index table, so each table gets its own task and is submitted to @@ -201,22 +201,22 @@ public Void call() throws Exception { HTableInterface table = factory.getTable(tableReference.get()); throwFailureIfDone(); table.batch(mutations); - } catch (CannotReachSingleIndexException e) { + } catch (SingleIndexWriteFailureException e) { throw e; } catch (IOException e) { - throw new CannotReachSingleIndexException(tableReference.toString(), mutations, e); + throw new SingleIndexWriteFailureException(tableReference.toString(), mutations, e); } catch (InterruptedException e) { // reset the interrupt status on the thread Thread.currentThread().interrupt(); - throw new CannotReachSingleIndexException(tableReference.toString(), mutations, e); + throw new SingleIndexWriteFailureException(tableReference.toString(), mutations, e); } return null; } - private void throwFailureIfDone() throws CannotReachSingleIndexException { + private void throwFailureIfDone() throws SingleIndexWriteFailureException { if (stopped.isStopped() || abortable.isAborted() || Thread.currentThread().isInterrupted()) { - throw new CannotReachSingleIndexException( + throw new SingleIndexWriteFailureException( "Pool closed, not attempting to write to the index!", null); } @@ -265,10 +265,11 @@ private void throwFailureIfDone() throws CannotReachSingleIndexException { // propagate the failure up to the caller try { this.abortable.throwCauseIfAborted(); - } catch (CannotReachSingleIndexException e) { + } catch (SingleIndexWriteFailureException e) { throw e; } catch (Throwable e) { - throw new CannotReachSingleIndexException("Got an abort notification while writing to the index!", + throw new SingleIndexWriteFailureException( + "Got an abort notification while writing to the index!", e); } diff --git a/src/main/java/com/salesforce/hbase/index/write/recovery/StoreFailuresInCachePolicy.java b/src/main/java/com/salesforce/hbase/index/write/recovery/StoreFailuresInCachePolicy.java index 0aebb363..bf0aae1c 100644 --- a/src/main/java/com/salesforce/hbase/index/write/recovery/StoreFailuresInCachePolicy.java +++ b/src/main/java/com/salesforce/hbase/index/write/recovery/StoreFailuresInCachePolicy.java @@ -25,14 +25,14 @@ import org.apache.hadoop.hbase.regionserver.HRegion; import com.google.common.collect.Multimap; -import com.salesforce.hbase.index.exception.MutliIndexWriteFailureException; +import com.salesforce.hbase.index.exception.MultiIndexWriteFailureException; import com.salesforce.hbase.index.table.HTableInterfaceReference; import com.salesforce.hbase.index.write.IndexFailurePolicy; import com.salesforce.hbase.index.write.KillServerOnFailurePolicy; /** * Tracks any failed writes in The {@link PerRegionIndexWriteCache}, given a - * {@link MutliIndexWriteFailureException} (which is thrown from the + * {@link MultiIndexWriteFailureException} (which is thrown from the * {@link TrackingParallelWriterIndexCommitter}. Any other exception failure causes the a server * abort via the usual {@link KillServerOnFailurePolicy}. */ @@ -60,11 +60,11 @@ public void setup(Stoppable parent, RegionCoprocessorEnvironment env) { @Override public void handleFailure(Multimap attempted, Exception cause) { // if its not an exception we can handle, let the delegate take care of it - if (!(cause instanceof MutliIndexWriteFailureException)) { + if (!(cause instanceof MultiIndexWriteFailureException)) { delegate.handleFailure(attempted, cause); } List failedTables = - ((MutliIndexWriteFailureException) cause).getFailedTables(); + ((MultiIndexWriteFailureException) cause).getFailedTables(); for (HTableInterfaceReference table : failedTables) { cache.addEdits(this.region, table, attempted.get(table)); } diff --git a/src/main/java/com/salesforce/hbase/index/write/recovery/TrackingParallelWriterIndexCommitter.java b/src/main/java/com/salesforce/hbase/index/write/recovery/TrackingParallelWriterIndexCommitter.java index 2dd03d4b..9176b7b7 100644 --- a/src/main/java/com/salesforce/hbase/index/write/recovery/TrackingParallelWriterIndexCommitter.java +++ b/src/main/java/com/salesforce/hbase/index/write/recovery/TrackingParallelWriterIndexCommitter.java @@ -54,8 +54,8 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.salesforce.hbase.index.CapturingAbortable; -import com.salesforce.hbase.index.exception.CannotReachSingleIndexException; -import com.salesforce.hbase.index.exception.MutliIndexWriteFailureException; +import com.salesforce.hbase.index.exception.MultiIndexWriteFailureException; +import com.salesforce.hbase.index.exception.SingleIndexWriteFailureException; import com.salesforce.hbase.index.table.CachingHTableFactory; import com.salesforce.hbase.index.table.HTableFactory; import com.salesforce.hbase.index.table.HTableInterfaceReference; @@ -69,10 +69,10 @@ * be a lot slower, in the face of failures, when compared to the * {@link ParallelWriterIndexCommitter} (though as fast for writes), so it should be used only when * you need to at least attempt all writes and know their result; for instance, this is fine for - * doing WAL recovery - its not a performance intensive situation and we want to limit the the edits - * we need to retry. + * doing WAL recovery - it's not a performance intensive situation and we want to limit the the + * edits we need to retry. *

    - * On failure to {@link #write(Multimap)}, we return a {@link MutliIndexWriteFailureException} that + * On failure to {@link #write(Multimap)}, we return a {@link MultiIndexWriteFailureException} that * contains the list of {@link HTableInterfaceReference} that didn't complete successfully. *

    * Failures to write to the index can happen several different ways: @@ -118,7 +118,7 @@ void setup(HTableFactory factory, ExecutorService pool, Abortable abortable, Sto @Override public void write(Multimap toWrite) - throws MutliIndexWriteFailureException { + throws MultiIndexWriteFailureException { // track the failures. We synchronize access to the failures via the basics synchronization // mechanisms. Its faster than using a fully concurrent list since we only need to synchronize // on add. This list is never exposed until all tasks have completed, at which point it is @@ -189,10 +189,10 @@ public Void call() throws Exception { return null; } - private void throwFailureIfDone() throws CannotReachSingleIndexException { + private void throwFailureIfDone() throws SingleIndexWriteFailureException { if (stopped.isStopped() || abortable.isAborted() || Thread.currentThread().isInterrupted()) { - throw new CannotReachSingleIndexException( + throw new SingleIndexWriteFailureException( "Pool closed, not attempting to write to the index!", null); } @@ -247,7 +247,7 @@ private void addToFailures(HTableInterfaceReference table) { // if any of the tasks failed, then we need to propagate the failure if (failures.size() > 0) { // make the list unmodifiable to avoid any more synchronization concerns - throw new MutliIndexWriteFailureException(Collections.unmodifiableList(failures)); + throw new MultiIndexWriteFailureException(Collections.unmodifiableList(failures)); } return; } diff --git a/src/test/java/com/salesforce/hbase/index/write/TestIndexWriter.java b/src/test/java/com/salesforce/hbase/index/write/TestIndexWriter.java index e1415fb2..3c241c9d 100644 --- a/src/test/java/com/salesforce/hbase/index/write/TestIndexWriter.java +++ b/src/test/java/com/salesforce/hbase/index/write/TestIndexWriter.java @@ -62,8 +62,8 @@ import com.salesforce.hbase.index.StubAbortable; import com.salesforce.hbase.index.TableName; -import com.salesforce.hbase.index.exception.CannotReachSingleIndexException; import com.salesforce.hbase.index.exception.IndexWriteException; +import com.salesforce.hbase.index.exception.SingleIndexWriteFailureException; import com.salesforce.hbase.index.util.ImmutableBytesPtr; public class TestIndexWriter { @@ -207,7 +207,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable { try { writer.write(indexUpdates); fail("Should not have successfully completed all index writes"); - } catch (CannotReachSingleIndexException e) { + } catch (SingleIndexWriteFailureException e) { LOG.info("Correctly got a failure to reach the index", e); // should have correctly gotten the correct abort, so let the next task execute waitOnAbortedLatch.countDown(); From 8b46d7519f0ebf63188d8528b3ceeb2b1ea36bb4 Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Tue, 1 Oct 2013 11:40:01 -0700 Subject: [PATCH 092/102] Fixing (un)compressed WAL Tests. Major issue was conflict between the IndexWALReader (which supports uncompressed WAL) and IndexWALEditCodec which supports compressed and uncompressed kvs. When the reader is installed, it can only support uncompressed WALEdits, which means using the Codec is unnecessary (and vice-versa). Therefore, in the current version, we only want to support the IndexWALEditCodec. However, we keed around the IndexWALReader to support 0.94.9, for those who want it (and updating the test to indicate that). --- .../hbase/index/IndexTestingUtils.java | 3 +-- ...estWALReplayWithCompressedIndexWrites.java | 25 ------------------ ...eplayWithIndexWritesAndCompressedWAL.java} | 17 +++++++++--- ...WritesAndUncompressedWALInHBase_094_9.java | 21 +++++++++++++++ ...WALReplayWithoutCompressedIndexWrites.java | 26 ------------------- 5 files changed, 35 insertions(+), 57 deletions(-) delete mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithCompressedIndexWrites.java rename src/test/java/org/apache/hadoop/hbase/regionserver/wal/{TestWALReplayWithIndexWrites.java => TestWALReplayWithIndexWritesAndCompressedWAL.java} (93%) create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWritesAndUncompressedWALInHBase_094_9.java delete mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithoutCompressedIndexWrites.java diff --git a/src/test/java/com/salesforce/hbase/index/IndexTestingUtils.java b/src/test/java/com/salesforce/hbase/index/IndexTestingUtils.java index ba569920..9da95027 100644 --- a/src/test/java/com/salesforce/hbase/index/IndexTestingUtils.java +++ b/src/test/java/com/salesforce/hbase/index/IndexTestingUtils.java @@ -66,8 +66,7 @@ private IndexTestingUtils() { public static void setupConfig(Configuration conf) { conf.setInt(MASTER_INFO_PORT_KEY, -1); conf.setInt(RS_INFO_PORT_KEY, -1); - // setup our codec and reader, so we get proper replay/write - conf.set(IndexManagementUtil.HLOG_READER_IMPL_KEY, IndexedHLogReader.class.getName()); + // setup our codec, so we get proper replay/write conf.set(WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY, IndexedWALEditCodec.class.getName()); } /** diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithCompressedIndexWrites.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithCompressedIndexWrites.java deleted file mode 100644 index 06460651..00000000 --- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithCompressedIndexWrites.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.apache.hadoop.hbase.regionserver.wal; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HConstants; -import org.junit.BeforeClass; - -/** - * Do the WAL Replay test but with the WALEditCodec, rather than an {@link IndexedHLogReader}, but - * still without compression - */ -public class TestWALReplayWithCompressedIndexWrites extends TestWALReplayWithIndexWrites { - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - configureCluster(); - // use our custom Codec to handle the custom WALEdits - Configuration conf = UTIL.getConfiguration(); - conf.set(WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY, IndexedWALEditCodec.class.getName()); - - // enable WAL compression - conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true); - - startCluster(); - } -} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWrites.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWritesAndCompressedWAL.java similarity index 93% rename from src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWrites.java rename to src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWritesAndCompressedWAL.java index 4935a2e1..1a3e54ec 100644 --- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWrites.java +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWritesAndCompressedWAL.java @@ -44,11 +44,15 @@ import com.salesforce.hbase.index.covered.example.CoveredColumnIndexer; /** - * most of the underlying work (creating/splitting the WAL, etc) is from + * For pre-0.94.9 instances, this class tests correctly deserializing WALEdits w/o compression. Post + * 0.94.9 we can support a custom {@link WALEditCodec}, which handles reading/writing the compressed + * edits. + *

    + * Most of the underlying work (creating/splitting the WAL, etc) is from * org.apache.hadoop.hhbase.regionserver.wal.TestWALReplay, copied here for completeness and ease of * use */ -public class TestWALReplayWithIndexWrites { +public class TestWALReplayWithIndexWritesAndCompressedWAL { public static final Log LOG = LogFactory.getLog(TestWALReplay.class); static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); @@ -90,8 +94,13 @@ protected static void startCluster() throws Exception { @BeforeClass public static void setUpBeforeClass() throws Exception { configureCluster(); - // use our custom WAL Reader - UTIL.getConfiguration().set("hbase.regionserver.hlog.reader.impl", IndexedHLogReader.class.getName()); + // use our custom Codec to handle the custom WALEdits + Configuration conf = UTIL.getConfiguration(); + conf.set(WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY, IndexedWALEditCodec.class.getName()); + + // enable WAL compression + conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true); + startCluster(); } diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWritesAndUncompressedWALInHBase_094_9.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWritesAndUncompressedWALInHBase_094_9.java new file mode 100644 index 00000000..1469283d --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWritesAndUncompressedWALInHBase_094_9.java @@ -0,0 +1,21 @@ +package org.apache.hadoop.hbase.regionserver.wal; + +import org.junit.BeforeClass; + +import com.salesforce.hbase.index.util.IndexManagementUtil; + +/** + * Do the WAL Replay test but with the WALEditCodec, rather than an {@link IndexedHLogReader}, but + * still with compression + */ +public class TestWALReplayWithIndexWritesAndUncompressedWALInHBase_094_9 extends TestWALReplayWithIndexWritesAndCompressedWAL { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + configureCluster(); + // use our custom WAL Reader + UTIL.getConfiguration().set(IndexManagementUtil.HLOG_READER_IMPL_KEY, + IndexedHLogReader.class.getName()); + startCluster(); + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithoutCompressedIndexWrites.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithoutCompressedIndexWrites.java deleted file mode 100644 index 20c32370..00000000 --- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithoutCompressedIndexWrites.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.apache.hadoop.hbase.regionserver.wal; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HConstants; -import org.junit.BeforeClass; - -/** - * Do the WAL Replay test but with the our custom {@link WALEditCodec} - {@link IndexedWALEditCodec} - * - and enabling compression - the main use case for having a custom {@link WALEditCodec}. - */ -public class TestWALReplayWithoutCompressedIndexWrites extends TestWALReplayWithIndexWrites { - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - configureCluster(); - - // use our custom Codec to handle the custom WALEdits - Configuration conf = UTIL.getConfiguration(); - conf.set(WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY, IndexedWALEditCodec.class.getName()); - - // disable WAL compression - conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, false); - - startCluster(); - } -} \ No newline at end of file From 6902758363a97982d81763304baa2ef5f929d59a Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Tue, 1 Oct 2013 13:53:22 -0700 Subject: [PATCH 093/102] Attempt test fix for TestWALReplayWithIndexWritesAndCompressedWAL --- .../wal/TestWALReplayWithIndexWritesAndCompressedWAL.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWritesAndCompressedWAL.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWritesAndCompressedWAL.java index 1a3e54ec..6868e123 100644 --- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWritesAndCompressedWAL.java +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWritesAndCompressedWAL.java @@ -42,6 +42,7 @@ import com.salesforce.hbase.index.covered.example.CoveredColumn; import com.salesforce.hbase.index.covered.example.CoveredColumnIndexSpecifierBuilder; import com.salesforce.hbase.index.covered.example.CoveredColumnIndexer; +import com.salesforce.hbase.index.util.IndexManagementUtil; /** * For pre-0.94.9 instances, this class tests correctly deserializing WALEdits w/o compression. Post @@ -76,6 +77,8 @@ protected static void configureCluster() throws Exception { conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 30000); conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 5000); IndexTestingUtils.setupConfig(conf); + // make sure we reset the WAL reader, as subclasses can mess with it potentially + conf.set(IndexManagementUtil.HLOG_READER_IMPL_KEY, SequenceFileLogReader.class.getName()); // enable appends conf.setBoolean("dfs.support.append", true); From 3bd7c291f4af3da9ac3859c0ccbec894fb259d17 Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Tue, 1 Oct 2013 16:21:37 -0700 Subject: [PATCH 094/102] Fix WALReplay tests. Problem was that surefire was running tests in the same JVM and HLog uses a static to track the logReader class. It was being set to the IndexedHLogReader in the _9 test, but then not reset in the regular test, causing the failure --- .../hbase/index/util/IndexManagementUtil.java | 21 +++-- .../regionserver/wal/IndexedHLogReader.java | 7 ++ ...ReplayWithIndexWritesAndCompressedWAL.java | 76 ++++++++++--------- ...WritesAndUncompressedWALInHBase_094_9.java | 19 +++-- 4 files changed, 75 insertions(+), 48 deletions(-) diff --git a/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java b/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java index 64d78090..93bba49f 100644 --- a/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java +++ b/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java @@ -33,6 +33,7 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.regionserver.wal.IndexedHLogReader; import org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec; @@ -66,11 +67,21 @@ public static void ensureMutableIndexingCorrectlyConfigured(Configuration conf) + " was not installed. You need to install it in hbase-site.xml under " + WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY); // ensure the Hlog Reader is correct - Preconditions.checkState( - conf.getClass(HLOG_READER_IMPL_KEY, IndexedHLogReader.class) == IndexedHLogReader.class, - IndexedHLogReader.class.getName() - + " was not installed. You need to install it in hbase-site.xml under " - + HLOG_READER_IMPL_KEY); + String indexLogReaderName = IndexedHLogReader.class.getName(); + if (conf.getBoolean(HConstants.ENABLE_WAL_COMPRESSION, false)) { + Preconditions.checkState( + conf.get(HLOG_READER_IMPL_KEY, indexLogReaderName).equals(indexLogReaderName), + indexLogReaderName + " was not installed. You need to install it in hbase-site.xml under " + + HLOG_READER_IMPL_KEY); + } else { + // can't support IndexHLogReader if WAL_COMPRESSION is enabled + Preconditions + .checkState( + !conf.get(HLOG_READER_IMPL_KEY, indexLogReaderName).equals(indexLogReaderName), + indexLogReaderName + + " was installed, but WAL_COMPRESSION wasn't enabled! You need to remove it in hbase-site.xml under " + + HLOG_READER_IMPL_KEY); + } } public static ValueGetter createGetterFromKeyValues(Collection pendingUpdates) { diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/IndexedHLogReader.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/IndexedHLogReader.java index 904b44c0..8fa7bfd0 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/IndexedHLogReader.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/IndexedHLogReader.java @@ -1,7 +1,10 @@ package org.apache.hadoop.hbase.regionserver.wal; import java.io.IOException; +import java.util.Arrays; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -29,6 +32,7 @@ * edits out into their respective regions. */ public class IndexedHLogReader implements Reader { + private static final Log LOG = LogFactory.getLog(IndexedHLogReader.class); private SequenceFileLogReader delegate; @@ -73,6 +77,9 @@ public void init(final FileSystem fs, final Path path, Configuration conf) throw // close the old reader and replace with our own, custom one this.delegate.reader.close(); this.delegate.reader = new IndexedWALReader(fs, path, conf); + Exception e = new Exception(); + LOG.info("Instantiated indexed log reader." + Arrays.toString(e.getStackTrace())); + LOG.info("Got conf: " + conf); } @Override diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWritesAndCompressedWAL.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWritesAndCompressedWAL.java index 6868e123..847976d4 100644 --- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWritesAndCompressedWAL.java +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWritesAndCompressedWAL.java @@ -51,24 +51,51 @@ *

    * Most of the underlying work (creating/splitting the WAL, etc) is from * org.apache.hadoop.hhbase.regionserver.wal.TestWALReplay, copied here for completeness and ease of - * use + * use. + *

    + * This test should only have a sinlge test - otherwise we will start/stop the minicluster multiple + * times, which is probably not what you want to do (mostly because its so much effort). */ public class TestWALReplayWithIndexWritesAndCompressedWAL { public static final Log LOG = LogFactory.getLog(TestWALReplay.class); - static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); @Rule public TableName table = new TableName(); private String INDEX_TABLE_NAME = table.getTableNameString() + "_INDEX"; + final HBaseTestingUtility UTIL = new HBaseTestingUtility(); private Path hbaseRootDir = null; private Path oldLogDir; private Path logDir; private FileSystem fs; private Configuration conf; - protected static void configureCluster() throws Exception { + @Before + public void setUp() throws Exception { + setupCluster(); + this.conf = HBaseConfiguration.create(UTIL.getConfiguration()); + this.fs = UTIL.getDFSCluster().getFileSystem(); + this.hbaseRootDir = new Path(this.conf.get(HConstants.HBASE_DIR)); + this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME); + this.logDir = new Path(this.hbaseRootDir, HConstants.HREGION_LOGDIR_NAME); + // reset the log reader to ensure we pull the one from this config + HLog.resetLogReaderClass(); + } + + private void setupCluster() throws Exception { + configureCluster(); + startCluster(); + } + + protected void configureCluster() throws Exception { Configuration conf = UTIL.getConfiguration(); + setDefaults(conf); + + // enable WAL compression + conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true); + } + + protected final void setDefaults(Configuration conf) { // make sure writers fail quickly conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3); conf.setInt(HConstants.HBASE_CLIENT_PAUSE, 1000); @@ -76,15 +103,12 @@ protected static void configureCluster() throws Exception { conf.setInt("zookeeper.recovery.retry.intervalmill", 100); conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 30000); conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 5000); - IndexTestingUtils.setupConfig(conf); - // make sure we reset the WAL reader, as subclasses can mess with it potentially - conf.set(IndexManagementUtil.HLOG_READER_IMPL_KEY, SequenceFileLogReader.class.getName()); - // enable appends conf.setBoolean("dfs.support.append", true); + IndexTestingUtils.setupConfig(conf); } - protected static void startCluster() throws Exception { + protected void startCluster() throws Exception { UTIL.startMiniDFSCluster(3); UTIL.startMiniZKCluster(); UTIL.startMiniHBaseCluster(1, 1); @@ -94,38 +118,13 @@ protected static void startCluster() throws Exception { UTIL.getConfiguration().set(HConstants.HBASE_DIR, hbaseRootDir.toString()); } - @BeforeClass - public static void setUpBeforeClass() throws Exception { - configureCluster(); - // use our custom Codec to handle the custom WALEdits - Configuration conf = UTIL.getConfiguration(); - conf.set(WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY, IndexedWALEditCodec.class.getName()); - - // enable WAL compression - conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true); - - startCluster(); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { + @After + public void tearDown() throws Exception { UTIL.shutdownMiniHBaseCluster(); UTIL.shutdownMiniDFSCluster(); UTIL.shutdownMiniZKCluster(); } - @Before - public void setUp() throws Exception { - this.conf = HBaseConfiguration.create(UTIL.getConfiguration()); - this.fs = UTIL.getDFSCluster().getFileSystem(); - this.hbaseRootDir = new Path(this.conf.get(HConstants.HBASE_DIR)); - this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME); - this.logDir = new Path(this.hbaseRootDir, HConstants.HREGION_LOGDIR_NAME); - } - - @After - public void tearDown() throws Exception { - } private void deleteDir(final Path p) throws IOException { if (this.fs.exists(p)) { @@ -142,6 +141,9 @@ private void deleteDir(final Path p) throws IOException { */ @Test public void testReplayEditsWrittenViaHRegion() throws Exception { + LOG.info("Jesse: Specified log reader:" + conf.get(IndexManagementUtil.HLOG_READER_IMPL_KEY)); + LOG.info("Jesse: Compression: " + conf.get(HConstants.ENABLE_WAL_COMPRESSION)); + LOG.info("Jesse: Codec: " + conf.get(WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY)); final String tableNameStr = "testReplayEditsWrittenViaHRegion"; final HRegionInfo hri = new HRegionInfo(Bytes.toBytes(tableNameStr), null, null, false); final Path basedir = new Path(this.hbaseRootDir, tableNameStr); @@ -247,6 +249,10 @@ private HLog createWAL(final Configuration c) throws IOException { */ private Path runWALSplit(final Configuration c) throws IOException { FileSystem fs = FileSystem.get(c); + LOG.info("Jesse: runWALSplit: hlogImpl: " + c.get("hbase.regionserver.hlog.reader.impl")); + LOG.info("Jesse: runWALSplit: hlogImpl: " + c.get(IndexManagementUtil.HLOG_READER_IMPL_KEY)); + LOG.info("Jesse: runWALSplit: Compression: " + conf.get(HConstants.ENABLE_WAL_COMPRESSION)); + LOG.info("Jesse: runWALSplit: Codec: " + conf.get(WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY)); HLogSplitter logSplitter = HLogSplitter.createLogSplitter(c, this.hbaseRootDir, this.logDir, this.oldLogDir, fs); List splits = logSplitter.splitLog(); diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWritesAndUncompressedWALInHBase_094_9.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWritesAndUncompressedWALInHBase_094_9.java index 1469283d..1395d969 100644 --- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWritesAndUncompressedWALInHBase_094_9.java +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplayWithIndexWritesAndUncompressedWALInHBase_094_9.java @@ -1,6 +1,7 @@ package org.apache.hadoop.hbase.regionserver.wal; -import org.junit.BeforeClass; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; import com.salesforce.hbase.index.util.IndexManagementUtil; @@ -10,12 +11,14 @@ */ public class TestWALReplayWithIndexWritesAndUncompressedWALInHBase_094_9 extends TestWALReplayWithIndexWritesAndCompressedWAL { - @BeforeClass - public static void setUpBeforeClass() throws Exception { - configureCluster(); - // use our custom WAL Reader - UTIL.getConfiguration().set(IndexManagementUtil.HLOG_READER_IMPL_KEY, - IndexedHLogReader.class.getName()); - startCluster(); + @Override + protected void configureCluster() throws Exception { + Configuration conf = UTIL.getConfiguration(); + setDefaults(conf); + LOG.info("Setting HLog impl to indexed log reader"); + conf.set(IndexManagementUtil.HLOG_READER_IMPL_KEY, IndexedHLogReader.class.getName()); + + // disable WAL compression + conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, false); } } \ No newline at end of file From 9815a5416368804d748a9beae1f64460b387f488 Mon Sep 17 00:00:00 2001 From: Jesse Yates Date: Tue, 1 Oct 2013 17:59:11 -0700 Subject: [PATCH 095/102] fixing enabling indexing + test --- .../hbase/index/util/IndexManagementUtil.java | 37 +++++---- .../index/util/TestIndexManagementUtil.java | 76 +++++++++++++++++++ 2 files changed, 93 insertions(+), 20 deletions(-) create mode 100644 src/test/java/com/salesforce/hbase/index/util/TestIndexManagementUtil.java diff --git a/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java b/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java index 93bba49f..5fe2df45 100644 --- a/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java +++ b/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java @@ -59,29 +59,26 @@ private IndexManagementUtil() { public static void ensureMutableIndexingCorrectlyConfigured(Configuration conf) throws IllegalStateException { - // ensure the WALEditCodec is correct - Preconditions - .checkState( - conf.getClass(WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY, IndexedWALEditCodec.class) == IndexedWALEditCodec.class, - IndexedWALEditCodec.class.getName() - + " was not installed. You need to install it in hbase-site.xml under " - + WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY); - // ensure the Hlog Reader is correct + // check to see if the WALEditCodec is installed + String codecClass = IndexedWALEditCodec.class.getName(); + if (codecClass.equals(conf.get(WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY, null))) { + // its installed, and it can handle compression and non-compression cases + return; + } + + // otherwise, we have to install the indexedhlogreader, but it cannot have compression String indexLogReaderName = IndexedHLogReader.class.getName(); - if (conf.getBoolean(HConstants.ENABLE_WAL_COMPRESSION, false)) { - Preconditions.checkState( - conf.get(HLOG_READER_IMPL_KEY, indexLogReaderName).equals(indexLogReaderName), - indexLogReaderName + " was not installed. You need to install it in hbase-site.xml under " - + HLOG_READER_IMPL_KEY); + if (indexLogReaderName.equals(conf.get(HLOG_READER_IMPL_KEY, indexLogReaderName))) { + if (conf.getBoolean(HConstants.ENABLE_WAL_COMPRESSION, false)) { + throw new IllegalStateException("WAL Compression is only supported with " + codecClass + + ". You can install in hbase-site.xml, under " + WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY); + } } else { - // can't support IndexHLogReader if WAL_COMPRESSION is enabled - Preconditions - .checkState( - !conf.get(HLOG_READER_IMPL_KEY, indexLogReaderName).equals(indexLogReaderName), - indexLogReaderName - + " was installed, but WAL_COMPRESSION wasn't enabled! You need to remove it in hbase-site.xml under " - + HLOG_READER_IMPL_KEY); + throw new IllegalStateException(IndexedWALEditCodec.class.getName() + + " is not installed, but " + indexLogReaderName + + " hasn't been installed in hbase-site.xml under " + HLOG_READER_IMPL_KEY); } + } public static ValueGetter createGetterFromKeyValues(Collection pendingUpdates) { diff --git a/src/test/java/com/salesforce/hbase/index/util/TestIndexManagementUtil.java b/src/test/java/com/salesforce/hbase/index/util/TestIndexManagementUtil.java new file mode 100644 index 00000000..183882ac --- /dev/null +++ b/src/test/java/com/salesforce/hbase/index/util/TestIndexManagementUtil.java @@ -0,0 +1,76 @@ +/******************************************************************************* + * Copyright (c) 2013, Salesforce.com, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of Salesforce.com nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.salesforce.hbase.index.util; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.regionserver.wal.IndexedHLogReader; +import org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec; +import org.apache.hadoop.hbase.regionserver.wal.WALEditCodec; +import org.junit.Test; + +public class TestIndexManagementUtil { + + @Test + public void testUncompressedWal() throws Exception { + Configuration conf = new Configuration(false); + // works with WALEditcodec + conf.set(WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY, IndexedWALEditCodec.class.getName()); + IndexManagementUtil.ensureMutableIndexingCorrectlyConfigured(conf); + // clear the codec and set the wal reader + conf = new Configuration(false); + conf.set(IndexManagementUtil.HLOG_READER_IMPL_KEY, IndexedHLogReader.class.getName()); + IndexManagementUtil.ensureMutableIndexingCorrectlyConfigured(conf); + } + + /** + * Compressed WALs are supported when we have the WALEditCodec installed + * @throws Exception + */ + @Test + public void testCompressedWALWithCodec() throws Exception { + Configuration conf = new Configuration(false); + conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true); + // works with WALEditcodec + conf.set(WALEditCodec.WAL_EDIT_CODEC_CLASS_KEY, IndexedWALEditCodec.class.getName()); + IndexManagementUtil.ensureMutableIndexingCorrectlyConfigured(conf); + } + + /** + * We cannot support WAL Compression with the IndexedHLogReader + * @throws Exception + */ + @Test(expected = IllegalStateException.class) + public void testCompressedWALWithHLogReader() throws Exception { + Configuration conf = new Configuration(false); + conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true); + // works with WALEditcodec + conf.set(IndexManagementUtil.HLOG_READER_IMPL_KEY, IndexedHLogReader.class.getName()); + IndexManagementUtil.ensureMutableIndexingCorrectlyConfigured(conf); + } +} \ No newline at end of file From 65d40e43fbfa08d04ebb6e33838a6447d41492d5 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Tue, 1 Oct 2013 21:39:03 -0700 Subject: [PATCH 096/102] Commonizing logic to create scan --- .../java/com/salesforce/hbase/index/Indexer.java | 1 + .../hbase/index/covered/data/LocalTable.java | 12 +++++------- .../hbase/index/util/IndexManagementUtil.java | 14 ++++++++++++++ .../salesforce/phoenix/index/IndexMaintainer.java | 7 ++++++- .../phoenix/index/PhoenixIndexBuilder.java | 15 ++------------- 5 files changed, 28 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/salesforce/hbase/index/Indexer.java b/src/main/java/com/salesforce/hbase/index/Indexer.java index c9405531..922acb93 100644 --- a/src/main/java/com/salesforce/hbase/index/Indexer.java +++ b/src/main/java/com/salesforce/hbase/index/Indexer.java @@ -312,6 +312,7 @@ public MultiMutation(ImmutableBytesPtr rowkey, boolean writeToWal) { /** * @param stored */ + @SuppressWarnings("deprecation") public void addAll(Mutation stored) { // add all the kvs for (Entry> kvs : stored.getFamilyMap().entrySet()) { diff --git a/src/main/java/com/salesforce/hbase/index/covered/data/LocalTable.java b/src/main/java/com/salesforce/hbase/index/covered/data/LocalTable.java index 87871c40..28ad96b9 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/data/LocalTable.java +++ b/src/main/java/com/salesforce/hbase/index/covered/data/LocalTable.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import org.apache.hadoop.hbase.KeyValue; @@ -41,6 +42,7 @@ import org.apache.hadoop.hbase.regionserver.RegionScanner; import com.salesforce.hbase.index.covered.update.ColumnReference; +import com.salesforce.hbase.index.util.IndexManagementUtil; /** * Wrapper around a lazily instantiated, local HTable. @@ -64,13 +66,9 @@ public Result getCurrentRowState(Mutation m, Collection kvs = new ArrayList(); diff --git a/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java b/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java index 64d78090..1eedc1d3 100644 --- a/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java +++ b/src/main/java/com/salesforce/hbase/index/util/IndexManagementUtil.java @@ -34,6 +34,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.regionserver.wal.IndexedHLogReader; import org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec; import org.apache.hadoop.hbase.regionserver.wal.WALEditCodec; @@ -183,4 +184,17 @@ public static boolean columnMatchesUpdate( } return matches; } + + public static Scan newLocalStateScan(List> refsArray) { + Scan s = new Scan(); + s.setRaw(true); + //add the necessary columns to the scan + for (Iterable refs : refsArray) { + for(ColumnReference ref : refs){ + s.addFamily(ref.getFamily()); + } + } + s.setMaxVersions(); + return s; + } } \ No newline at end of file diff --git a/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java b/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java index 287a4284..d4c7ea02 100644 --- a/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java +++ b/src/main/java/com/salesforce/phoenix/index/IndexMaintainer.java @@ -58,7 +58,7 @@ * @author jtaylor * @since 2.1.0 */ -public class IndexMaintainer implements Writable { +public class IndexMaintainer implements Writable, Iterable { public static IndexMaintainer create(PTable dataTable, PTable index) { if (dataTable.getType() == PTableType.INDEX || index.getType() != PTableType.INDEX || !dataTable.getIndexes().contains(index)) { @@ -692,4 +692,9 @@ public void readFields(DataInput input) throws IOException { } } } + + @Override + public Iterator iterator() { + return allColumns.iterator(); + } } diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java index 92cd7656..243e7318 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java @@ -31,7 +31,7 @@ import com.google.common.collect.Lists; import com.salesforce.hbase.index.covered.CoveredColumnsIndexBuilder; -import com.salesforce.hbase.index.covered.update.ColumnReference; +import com.salesforce.hbase.index.util.IndexManagementUtil; import com.salesforce.phoenix.compile.ScanRanges; import com.salesforce.phoenix.query.KeyRange; import com.salesforce.phoenix.schema.PDataType; @@ -55,18 +55,7 @@ public void batchStarted(MiniBatchOperationInProgress> m maintainers.addAll(getCodec().getIndexMaintainers(m.getAttributesMap())); } ScanRanges scanRanges = ScanRanges.create(Collections.singletonList(keys), SchemaUtil.VAR_BINARY_SCHEMA); - Scan scan = new Scan(); - scan.setRaw(true); - // Project into scan only the columns we need to build the new index row and - // delete the old index row. We use the columns that we pass through for - // the Delete use case, as it includes indexed and covered columns as well - // as the empty key value column (which we use to detect a delete of the entire row). - for (int i = 0; i < maintainers.size(); i++) { - IndexMaintainer maintainer = maintainers.get(i); - for (ColumnReference ref : maintainer.getAllColumns()) { - scan.addFamily(ref.getFamily()); - } - } + Scan scan = IndexManagementUtil.newLocalStateScan(maintainers); scan.setFilter(scanRanges.getSkipScanFilter()); HRegion region = this.env.getRegion(); RegionScanner scanner = region.getScanner(scan); From 9c6ddf405ad7d417fb0aa749ef6f15293cd97d97 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Wed, 2 Oct 2013 00:28:43 -0700 Subject: [PATCH 097/102] Set start/stop of skip scan, size array list expected to be 1 --- .../com/salesforce/hbase/index/covered/data/LocalTable.java | 4 ++-- .../com/salesforce/phoenix/index/PhoenixIndexBuilder.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/salesforce/hbase/index/covered/data/LocalTable.java b/src/main/java/com/salesforce/hbase/index/covered/data/LocalTable.java index 28ad96b9..26048b88 100644 --- a/src/main/java/com/salesforce/hbase/index/covered/data/LocalTable.java +++ b/src/main/java/com/salesforce/hbase/index/covered/data/LocalTable.java @@ -68,10 +68,10 @@ public Result getCurrentRowState(Mutation m, Collection kvs = new ArrayList(); + List kvs = new ArrayList(1); boolean more = scanner.next(kvs); assert !more : "Got more than one result when scanning" + " a single row in the primary table!"; diff --git a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java index 243e7318..1c8b335e 100644 --- a/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java +++ b/src/main/java/com/salesforce/phoenix/index/PhoenixIndexBuilder.java @@ -54,8 +54,9 @@ public void batchStarted(MiniBatchOperationInProgress> m keys.add(PDataType.VARBINARY.getKeyRange(m.getRow())); maintainers.addAll(getCodec().getIndexMaintainers(m.getAttributesMap())); } - ScanRanges scanRanges = ScanRanges.create(Collections.singletonList(keys), SchemaUtil.VAR_BINARY_SCHEMA); Scan scan = IndexManagementUtil.newLocalStateScan(maintainers); + ScanRanges scanRanges = ScanRanges.create(Collections.singletonList(keys), SchemaUtil.VAR_BINARY_SCHEMA); + scanRanges.setScanStartStopRow(scan); scan.setFilter(scanRanges.getSkipScanFilter()); HRegion region = this.env.getRegion(); RegionScanner scanner = region.getScanner(scan); From 523c91f0816133718a7eb4f788499a872316e6a9 Mon Sep 17 00:00:00 2001 From: maryannxue Date: Wed, 2 Oct 2013 12:37:04 -0400 Subject: [PATCH 098/102] bug fix --- .../phoenix/compile/JoinCompiler.java | 5 +- .../phoenix/compile/WhereCompiler.java | 23 --- .../coprocessor/HashJoinRegionScanner.java | 22 +-- .../ServerCachingEndpointImpl.java | 4 +- .../expression/KeyValueColumnExpression.java | 54 ++++++- .../expression/RowKeyColumnExpression.java | 60 ++----- .../filter/MultiKeyValueComparisonFilter.java | 36 +---- .../filter/RowKeyComparisonFilter.java | 5 - .../phoenix/join/HashCacheFactory.java | 2 +- .../phoenix/join/ScanProjector.java | 148 +++--------------- .../salesforce/phoenix/schema/ColumnRef.java | 2 +- .../phoenix/schema/RowKeyValueAccessor.java | 23 ++- .../schema/tuple/MultiKeyValueTuple.java | 19 +-- .../phoenix/schema/tuple/ResultTuple.java | 32 ++-- .../schema/tuple/SingleKeyValueTuple.java | 11 -- .../phoenix/schema/tuple/Tuple.java | 12 -- .../phoenix/end2end/HashJoinTest.java | 30 ++-- .../phoenix/end2end/OrderByTest.java | 2 +- .../salesforce/phoenix/query/BaseTest.java | 20 +-- 19 files changed, 166 insertions(+), 344 deletions(-) diff --git a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java index aa19407c..bd9ec4d1 100644 --- a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java @@ -43,7 +43,6 @@ import com.salesforce.phoenix.expression.AndExpression; import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.join.ScanProjector; -import com.salesforce.phoenix.join.ScanProjector.ProjectionType; import com.salesforce.phoenix.parse.AliasedNode; import com.salesforce.phoenix.parse.AndParseNode; import com.salesforce.phoenix.parse.CaseParseNode; @@ -182,7 +181,7 @@ public ScanProjector getScanProjector() { } else { tableName = mainTable.getTableName(); } - return new ScanProjector(ProjectionType.TABLE, tableName, null, null); + return new ScanProjector(tableName); } public List compilePostFilterExpressions(StatementContext context) throws SQLException { @@ -373,7 +372,7 @@ public SelectStatement getAsSubquery() { } public ScanProjector getScanProjector() { - return new ScanProjector(ProjectionType.TABLE, ScanProjector.getPrefixForTable(table), null, null); + return new ScanProjector(ScanProjector.getPrefixForTable(table)); } public List compilePostFilterExpressions(StatementContext context) throws SQLException { diff --git a/src/main/java/com/salesforce/phoenix/compile/WhereCompiler.java b/src/main/java/com/salesforce/phoenix/compile/WhereCompiler.java index 57888a01..487c9d5b 100644 --- a/src/main/java/com/salesforce/phoenix/compile/WhereCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/WhereCompiler.java @@ -144,23 +144,6 @@ public void increment(KeyValueColumnExpression column) { } } - - public void increment(RowKeyColumnExpression column) { - if (column.getCFPrefix() == null) - return; - - switch (count) { - case NONE: - count = Count.MULTIPLE; - break; - case SINGLE: - count = Count.MULTIPLE; - break; - case MULTIPLE: - break; - - } - } public Count getCount() { return count; @@ -197,12 +180,6 @@ public Void visit(KeyValueColumnExpression expression) { counter.increment(expression); return null; } - - @Override - public Void visit(RowKeyColumnExpression expression) { - counter.increment(expression); - return null; - } }); switch (counter.getCount()) { case NONE: diff --git a/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java b/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java index ce785d33..3bac2225 100644 --- a/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java +++ b/src/main/java/com/salesforce/phoenix/coprocessor/HashJoinRegionScanner.java @@ -65,8 +65,6 @@ public HashJoinRegionScanner(RegionScanner scanner, ScanProjector projector, Has this.resultQueue = new LinkedList>(); this.hasMore = true; if (joinInfo != null) { - if (tenantId == null) - throw new IOException("Could not find tenant id for hash cache."); for (JoinType type : joinInfo.getJoinTypes()) { if (type != JoinType.Inner && type != JoinType.Left) throw new IOException("Got join type '" + type + "'. Expect only INNER or LEFT with hash-joins."); @@ -81,9 +79,10 @@ private void processResults(List result, boolean hasLimit) throws IOEx return; if (projector != null) { - List kvs = new ArrayList(result.size()); + List kvs = new ArrayList(result.size() + 1); + kvs.add(projector.projectRow(result.get(0))); for (KeyValue kv : result) { - kvs.add(projector.getProjectedKeyValue(kv)); + kvs.add(projector.projectedKeyValue(kv)); } if (joinInfo != null) { result = kvs; @@ -105,7 +104,7 @@ private void processResults(List result, boolean hasLimit) throws IOEx ImmutableBytesPtr joinId = joinInfo.getJoinIds()[i]; HashCache hashCache = (HashCache)cache.getServerCache(joinId); if (hashCache == null) - throw new IOException("Could not find hash cache for joinId: " + Bytes.toString(joinId.get())); + throw new IOException("Could not find hash cache for joinId: " + Bytes.toString(joinId.get(), joinId.getOffset(), joinId.getLength())); tuples[i] = hashCache.get(key); JoinType type = joinInfo.getJoinTypes()[i]; if (type == JoinType.Inner && (tuples[i] == null || tuples[i].isEmpty())) { @@ -130,12 +129,13 @@ private void processResults(List result, boolean hasLimit) throws IOEx } } } - // apply post-join filter + // sort kvs and apply post-join filter Expression postFilter = joinInfo.getPostJoinFilterExpression(); - if (postFilter != null) { - ImmutableBytesWritable tempPtr = new ImmutableBytesWritable(); - for (Iterator> iter = resultQueue.iterator(); iter.hasNext();) { - Tuple t = new ResultTuple(new Result(iter.next())); + for (Iterator> iter = resultQueue.iterator(); iter.hasNext();) { + List kvs = iter.next(); + if (postFilter != null) { + Tuple t = new ResultTuple(new Result(kvs)); + ImmutableBytesWritable tempPtr = new ImmutableBytesWritable(); try { if (!postFilter.evaluate(t, tempPtr)) { iter.remove(); @@ -148,8 +148,10 @@ private void processResults(List result, boolean hasLimit) throws IOEx Boolean b = (Boolean)postFilter.getDataType().toObject(tempPtr); if (!b.booleanValue()) { iter.remove(); + continue; } } + Collections.sort(kvs, KeyValue.COMPARATOR); } } } diff --git a/src/main/java/com/salesforce/phoenix/coprocessor/ServerCachingEndpointImpl.java b/src/main/java/com/salesforce/phoenix/coprocessor/ServerCachingEndpointImpl.java index 4ead038e..6afe3513 100644 --- a/src/main/java/com/salesforce/phoenix/coprocessor/ServerCachingEndpointImpl.java +++ b/src/main/java/com/salesforce/phoenix/coprocessor/ServerCachingEndpointImpl.java @@ -50,14 +50,14 @@ public class ServerCachingEndpointImpl extends BaseEndpointCoprocessor implement @Override public boolean addServerCache(byte[] tenantId, byte[] cacheId, ImmutableBytesWritable cachePtr, ServerCacheFactory cacheFactory) throws SQLException { - TenantCache tenantCache = GlobalCache.getTenantCache(this.getEnvironment().getConfiguration(), new ImmutableBytesPtr(tenantId)); + TenantCache tenantCache = GlobalCache.getTenantCache(this.getEnvironment().getConfiguration(), tenantId == null ? null : new ImmutableBytesPtr(tenantId)); tenantCache.addServerCache(new ImmutableBytesPtr(cacheId), cachePtr, cacheFactory); return true; } @Override public boolean removeServerCache(byte[] tenantId, byte[] cacheId) throws SQLException { - TenantCache tenantCache = GlobalCache.getTenantCache(this.getEnvironment().getConfiguration(), new ImmutableBytesPtr(tenantId)); + TenantCache tenantCache = GlobalCache.getTenantCache(this.getEnvironment().getConfiguration(), tenantId == null ? null : new ImmutableBytesPtr(tenantId)); tenantCache.removeServerCache(new ImmutableBytesPtr(cacheId)); return true; } diff --git a/src/main/java/com/salesforce/phoenix/expression/KeyValueColumnExpression.java b/src/main/java/com/salesforce/phoenix/expression/KeyValueColumnExpression.java index b8f6f089..d77c469f 100644 --- a/src/main/java/com/salesforce/phoenix/expression/KeyValueColumnExpression.java +++ b/src/main/java/com/salesforce/phoenix/expression/KeyValueColumnExpression.java @@ -38,6 +38,7 @@ import com.salesforce.phoenix.join.ScanProjector; import com.salesforce.phoenix.query.QueryConstants; import com.salesforce.phoenix.schema.PColumn; +import com.salesforce.phoenix.schema.RowKeyValueAccessor; import com.salesforce.phoenix.schema.tuple.Tuple; @@ -51,6 +52,7 @@ public class KeyValueColumnExpression extends ColumnExpression { private byte[] cf; private byte[] cq; + private RowKeyValueAccessor accessor; public KeyValueColumnExpression() { } @@ -67,6 +69,13 @@ public KeyValueColumnExpression(PColumn column, byte[] cfPrefix) { this.cq = column.getName().getBytes(); } + public KeyValueColumnExpression(PColumn column, byte[] cfPrefix, RowKeyValueAccessor accessor) { + super(column); + this.cf = ScanProjector.getRowFamily(cfPrefix); + this.cq = ScanProjector.getRowQualifier(); + this.accessor = accessor; + } + public byte[] getColumnFamily() { return cf; } @@ -74,6 +83,10 @@ public byte[] getColumnFamily() { public byte[] getColumnName() { return cq; } + + public int getPosition() { + return accessor.getIndex(); + } @Override public int hashCode() { @@ -81,6 +94,7 @@ public int hashCode() { int result = 1; result = prime * result + Arrays.hashCode(cf); result = prime * result + Arrays.hashCode(cq); + result = prime * result + ((accessor == null) ? 0 : accessor.hashCode()); return result; } @@ -93,12 +107,15 @@ public boolean equals(Object obj) { KeyValueColumnExpression other = (KeyValueColumnExpression)obj; if (!Arrays.equals(cf, other.cf)) return false; if (!Arrays.equals(cq, other.cq)) return false; + if ((accessor != null && !accessor.equals(other.accessor)) + || (accessor == null && other.accessor != null)) + return false; return true; } @Override public String toString() { - return (Bytes.compareTo(cf, QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES) == 0 ? "" : (Bytes.toStringBinary(cf) + QueryConstants.NAME_SEPARATOR)) + Bytes.toStringBinary(cq); + return (Bytes.compareTo(cf, QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES) == 0 ? "" : (Bytes.toStringBinary(cf) + QueryConstants.NAME_SEPARATOR)) + Bytes.toStringBinary(cq) + (accessor == null ? "" : ("[" + accessor.getIndex() + "]")); } @Override @@ -106,7 +123,27 @@ public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { KeyValue keyValue = tuple.getValue(cf, cq); if (keyValue != null) { ptr.set(keyValue.getBuffer(), keyValue.getValueOffset(), keyValue.getValueLength()); - return true; + if (accessor == null) + return true; + int offset = accessor.getOffset(ptr.get(), ptr.getOffset()); + // Null is represented in the last expression of a multi-part key + // by the bytes not being present. + int maxOffset = ptr.getOffset() + ptr.getLength(); + if (offset < maxOffset) { + byte[] buffer = ptr.get(); + int fixedByteSize = -1; + // FIXME: fixedByteSize <= maxByteSize ? fixedByteSize : 0 required because HBase passes bogus keys to filter to position scan (HBASE-6562) + if (type.isFixedWidth()) { + fixedByteSize = getByteSize(); + fixedByteSize = fixedByteSize <= maxOffset ? fixedByteSize : 0; + } + int length = fixedByteSize >= 0 ? fixedByteSize : accessor.getLength(buffer, offset, maxOffset); + // In the middle of the key, an empty variable length byte array represents null + if (length > 0) { + ptr.set(type.toBytes(type.toObject(buffer, offset, length, type))); + return true; + } + } } return false; } @@ -116,6 +153,13 @@ public void readFields(DataInput input) throws IOException { super.readFields(input); cf = Bytes.readByteArray(input); cq = Bytes.readByteArray(input); + boolean hasAccessor = input.readBoolean(); + if (hasAccessor) { + accessor = new RowKeyValueAccessor(); + accessor.readFields(input); + } else { + accessor = null; + } } @Override @@ -123,6 +167,12 @@ public void write(DataOutput output) throws IOException { super.write(output); Bytes.writeByteArray(output, cf); Bytes.writeByteArray(output, cq); + if (accessor != null) { + output.writeBoolean(true); + accessor.write(output); + } else { + output.writeBoolean(false); + } } @Override diff --git a/src/main/java/com/salesforce/phoenix/expression/RowKeyColumnExpression.java b/src/main/java/com/salesforce/phoenix/expression/RowKeyColumnExpression.java index a7b677c8..6be6e2cf 100644 --- a/src/main/java/com/salesforce/phoenix/expression/RowKeyColumnExpression.java +++ b/src/main/java/com/salesforce/phoenix/expression/RowKeyColumnExpression.java @@ -27,13 +27,16 @@ ******************************************************************************/ package com.salesforce.phoenix.expression; -import java.io.*; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.util.Bytes; import com.salesforce.phoenix.expression.visitor.ExpressionVisitor; -import com.salesforce.phoenix.schema.*; +import com.salesforce.phoenix.schema.PDataType; +import com.salesforce.phoenix.schema.PDatum; +import com.salesforce.phoenix.schema.RowKeyValueAccessor; import com.salesforce.phoenix.schema.tuple.Tuple; @@ -47,49 +50,33 @@ public class RowKeyColumnExpression extends ColumnExpression { private PDataType fromType; private RowKeyValueAccessor accessor; - private byte[] cfPrefix; - private final String name; + protected final String name; public RowKeyColumnExpression() { name = null; // Only on client - cfPrefix = null; } public RowKeyColumnExpression(PDatum datum, RowKeyValueAccessor accessor) { this(datum, accessor, datum.getDataType()); } - public RowKeyColumnExpression(PDatum datum, RowKeyValueAccessor accessor, byte[] cfPrefix) { - this(datum, accessor, datum.getDataType(), cfPrefix); - } - public RowKeyColumnExpression(PDatum datum, RowKeyValueAccessor accessor, PDataType fromType) { - this(datum, accessor, fromType, null); - } - - public RowKeyColumnExpression(PDatum datum, RowKeyValueAccessor accessor, PDataType fromType, byte[] cfPrefix) { super(datum); this.accessor = accessor; this.fromType = fromType; this.name = datum.toString(); - this.cfPrefix = cfPrefix; } public int getPosition() { return accessor.getIndex(); } - - public byte[] getCFPrefix() { - return cfPrefix; - } @Override public int hashCode() { final int prime = 31; int result = super.hashCode(); result = prime * result + ((accessor == null) ? 0 : accessor.hashCode()); - result = prime * result + ((cfPrefix == null) ? 0 : Bytes.hashCode(cfPrefix)); return result; } @@ -104,30 +91,25 @@ public boolean equals(Object obj) { if (!super.equals(obj)) return false; if (getClass() != obj.getClass()) return false; RowKeyColumnExpression other = (RowKeyColumnExpression)obj; - return accessor.equals(other.accessor) - && ((cfPrefix == null && other.cfPrefix == null) - || (cfPrefix != null && other.cfPrefix != null && Bytes.equals(cfPrefix, other.cfPrefix))); + return accessor.equals(other.accessor); } @Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { - boolean success = getKey(tuple, ptr); - if (success == false) - return false; - + tuple.getKey(ptr); int offset = accessor.getOffset(ptr.get(), ptr.getOffset()); // Null is represented in the last expression of a multi-part key // by the bytes not being present. - if (offset < ptr.getOffset() + ptr.getLength()) { + int maxOffset = ptr.getOffset() + ptr.getLength(); + if (offset < maxOffset) { byte[] buffer = ptr.get(); - int maxByteSize = ptr.getLength() - (offset - ptr.getOffset()); int fixedByteSize = -1; // FIXME: fixedByteSize <= maxByteSize ? fixedByteSize : 0 required because HBase passes bogus keys to filter to position scan (HBASE-6562) if (fromType.isFixedWidth()) { fixedByteSize = getByteSize(); - fixedByteSize = fixedByteSize <= maxByteSize ? fixedByteSize : 0; + fixedByteSize = fixedByteSize <= maxOffset ? fixedByteSize : 0; } - int length = fixedByteSize >= 0 ? fixedByteSize : accessor.getLength(buffer, offset, maxByteSize); + int length = fixedByteSize >= 0 ? fixedByteSize : accessor.getLength(buffer, offset, maxOffset); // In the middle of the key, an empty variable length byte array represents null if (length > 0) { if (type == fromType) { @@ -140,15 +122,6 @@ public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { } return false; } - - protected boolean getKey(Tuple tuple, ImmutableBytesWritable ptr) { - if (cfPrefix == null) { - tuple.getKey(ptr); - return true; - } - - return tuple.getKey(ptr, cfPrefix); - } @Override public void readFields(DataInput input) throws IOException { @@ -156,19 +129,12 @@ public void readFields(DataInput input) throws IOException { accessor = new RowKeyValueAccessor(); accessor.readFields(input); fromType = type; // fromType only needed on client side - byte[] prefix = Bytes.readByteArray(input); - cfPrefix = prefix.length == 0 ? null : prefix; } @Override public void write(DataOutput output) throws IOException { super.write(output); accessor.write(output); - if (cfPrefix == null) { - Bytes.writeByteArray(output, new byte[0]); - } else { - Bytes.writeByteArray(output, cfPrefix); - } } @Override diff --git a/src/main/java/com/salesforce/phoenix/filter/MultiKeyValueComparisonFilter.java b/src/main/java/com/salesforce/phoenix/filter/MultiKeyValueComparisonFilter.java index a8140f38..2a7abd87 100644 --- a/src/main/java/com/salesforce/phoenix/filter/MultiKeyValueComparisonFilter.java +++ b/src/main/java/com/salesforce/phoenix/filter/MultiKeyValueComparisonFilter.java @@ -38,8 +38,6 @@ import com.salesforce.phoenix.expression.Expression; import com.salesforce.phoenix.expression.KeyValueColumnExpression; -import com.salesforce.phoenix.expression.RowKeyColumnExpression; -import com.salesforce.phoenix.join.ScanProjector; import com.salesforce.phoenix.schema.tuple.Tuple; @@ -54,7 +52,6 @@ */ public abstract class MultiKeyValueComparisonFilter extends BooleanExpressionFilter { private static final byte[] UNITIALIZED_KEY_BUFFER = new byte[0]; - private static final byte[] EMPTY_CQ = new byte[0]; private Boolean matchedColumn; protected final IncrementalResultTuple inputTuple = new IncrementalResultTuple(); @@ -106,24 +103,12 @@ public ReturnCode resolveColumn(KeyValue value) { // and our expression uses row key columns. setKey(value); byte[] buf = value.getBuffer(); - // check if this kv matches RowKeyColumnExpression with a cf-prefix. - ReturnCode ret = ReturnCode.INCLUDE; - int pLen = ScanProjector.getPrefixLength(buf, value.getFamilyOffset(), value.getFamilyLength()); - if (pLen != 0) { - Object prefixPtr = setColumnKey(buf, value.getFamilyOffset(), pLen, EMPTY_CQ, 0, 0); - KeyValueRef rowRef = foundColumns.get(prefixPtr); - if (rowRef != null && rowRef.keyValue == null) { - rowRef.keyValue = value; - refCount++; - ret = null; - } - } Object ptr = setColumnKey(buf, value.getFamilyOffset(), value.getFamilyLength(), buf, value.getQualifierOffset(), value.getQualifierLength()); KeyValueRef ref = foundColumns.get(ptr); if (ref == null) { // Return INCLUDE here. Although this filter doesn't need this KV // it should still be projected into the Result - return ret; + return ReturnCode.INCLUDE; } // Since we only look at the latest key value for a given column, // we are not interested in older versions @@ -186,17 +171,6 @@ public KeyValue getValue(int index) { } throw new IndexOutOfBoundsException(Integer.toString(index)); } - - @Override - public boolean getKey(ImmutableBytesWritable ptr, byte[] cfPrefix) { - Object key = setColumnKey(cfPrefix, 0, cfPrefix.length, EMPTY_CQ, 0, 0); - KeyValueRef ref = foundColumns.get(key); - if (ref == null || ref.keyValue == null) - return false; - - ptr.set(ref.keyValue.getBuffer(), ref.keyValue.getKeyOffset(), ref.keyValue.getKeyLength()); - return true; - } } protected void init() { @@ -206,14 +180,6 @@ public Void visit(KeyValueColumnExpression expression) { inputTuple.addColumn(expression.getColumnFamily(), expression.getColumnName()); return null; } - @Override - public Void visit(RowKeyColumnExpression expression) { - byte[] cfPrefix = expression.getCFPrefix(); - if (cfPrefix != null) { - inputTuple.addColumn(cfPrefix, EMPTY_CQ); - } - return null; - } }; expression.accept(visitor); this.evaluateOnCompletion = visitor.evaluateOnCompletion(); diff --git a/src/main/java/com/salesforce/phoenix/filter/RowKeyComparisonFilter.java b/src/main/java/com/salesforce/phoenix/filter/RowKeyComparisonFilter.java index 7c750a9d..9a6cc52b 100644 --- a/src/main/java/com/salesforce/phoenix/filter/RowKeyComparisonFilter.java +++ b/src/main/java/com/salesforce/phoenix/filter/RowKeyComparisonFilter.java @@ -115,11 +115,6 @@ public int size() { public KeyValue getValue(int index) { throw new IndexOutOfBoundsException(Integer.toString(index)); } - - @Override - public boolean getKey(ImmutableBytesWritable ptr, byte[] cfPrefix) { - return false; - } } @Override diff --git a/src/main/java/com/salesforce/phoenix/join/HashCacheFactory.java b/src/main/java/com/salesforce/phoenix/join/HashCacheFactory.java index 420a9c07..5ac7216c 100644 --- a/src/main/java/com/salesforce/phoenix/join/HashCacheFactory.java +++ b/src/main/java/com/salesforce/phoenix/join/HashCacheFactory.java @@ -106,7 +106,7 @@ private HashCacheImpl(byte[] hashCacheBytes, MemoryChunk memoryChunk) { offset += WritableUtils.decodeVIntSize(hashCacheByteArray[offset]); ImmutableBytesWritable value = new ImmutableBytesWritable(hashCacheByteArray,offset,resultSize); Tuple result = new ResultTuple(new Result(value)); - ImmutableBytesPtr key = new ImmutableBytesPtr(TupleUtil.getConcatenatedValue(result, onExpressions)); + ImmutableBytesPtr key = TupleUtil.getConcatenatedValue(result, onExpressions); List tuples = hashCacheMap.get(key); if (tuples == null) { tuples = new ArrayList(1); diff --git a/src/main/java/com/salesforce/phoenix/join/ScanProjector.java b/src/main/java/com/salesforce/phoenix/join/ScanProjector.java index 0fd3cb32..53bae35d 100644 --- a/src/main/java/com/salesforce/phoenix/join/ScanProjector.java +++ b/src/main/java/com/salesforce/phoenix/join/ScanProjector.java @@ -32,47 +32,34 @@ import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.io.WritableUtils; import com.salesforce.phoenix.schema.TableRef; import com.salesforce.phoenix.util.ByteUtil; -import com.salesforce.phoenix.util.ImmutableBytesPtr; import com.salesforce.phoenix.util.KeyValueUtil; -public class ScanProjector { - - public enum ProjectionType {TABLE, CF, CQ}; - +public class ScanProjector { private static final String SCAN_PROJECTOR = "scanProjector"; private static final byte[] PREFIX_SEPERATOR = Bytes.toBytes(":"); + private static final byte[] PROJECTED_ROW = Bytes.toBytes("_"); + private static final byte[] EMPTY_QUALIFIER = new byte[0]; - private final ProjectionType type; private final byte[] tablePrefix; - private final Map cfProjectionMap; - private final Map>> cqProjectionMap; - public static byte[] getPrefixedColumnFamily(byte[] columnFamily, byte[] cfPrefix) { - return ByteUtil.concat(cfPrefix, PREFIX_SEPERATOR, columnFamily); + public static byte[] getRowFamily(byte[] cfPrefix) { + return ByteUtil.concat(cfPrefix, PREFIX_SEPERATOR); } - public static int getPrefixLength(byte[] columnFamily) { - return getPrefixLength(columnFamily, 0, columnFamily.length); + public static byte[] getRowQualifier() { + return EMPTY_QUALIFIER; } - - public static int getPrefixLength(byte[] cfBuffer, int offset, int length) { - for (int i = offset + length - 1; i >= offset; i--) { - if (cfBuffer[i] == PREFIX_SEPERATOR[0]) { - return (i - offset); - } - } - return 0; + + public static byte[] getPrefixedColumnFamily(byte[] columnFamily, byte[] cfPrefix) { + return ByteUtil.concat(cfPrefix, PREFIX_SEPERATOR, columnFamily); } public static byte[] getPrefixForTable(TableRef table) { @@ -82,48 +69,15 @@ public static byte[] getPrefixForTable(TableRef table) { return Bytes.toBytes(table.getTableAlias()); } - public ScanProjector(ProjectionType type, byte[] tablePrefix, - Map cfProjectionMap, Map>> cqProjectionMap) { - this.type = ProjectionType.TABLE; + public ScanProjector(byte[] tablePrefix) { this.tablePrefix = tablePrefix; - this.cfProjectionMap = cfProjectionMap; - this.cqProjectionMap = cqProjectionMap; } public static void serializeProjectorIntoScan(Scan scan, ScanProjector projector) { ByteArrayOutputStream stream = new ByteArrayOutputStream(); try { DataOutputStream output = new DataOutputStream(stream); - WritableUtils.writeVInt(output, projector.type.ordinal()); - switch (projector.type) { - case TABLE: - WritableUtils.writeCompressedByteArray(output, projector.tablePrefix); - break; - case CF: - WritableUtils.writeVInt(output, projector.cfProjectionMap.size()); - for (Map.Entry entry : projector.cfProjectionMap.entrySet()) { - WritableUtils.writeCompressedByteArray(output, entry.getKey().get()); - WritableUtils.writeCompressedByteArray(output, entry.getValue()); - } - break; - case CQ: - WritableUtils.writeVInt(output, projector.cqProjectionMap.size()); - for (Map.Entry>> entry : - projector.cqProjectionMap.entrySet()) { - WritableUtils.writeCompressedByteArray(output, entry.getKey().get()); - Map> map = entry.getValue(); - WritableUtils.writeVInt(output, map.size()); - for (Map.Entry> e : map.entrySet()) { - WritableUtils.writeCompressedByteArray(output, e.getKey().get()); - WritableUtils.writeCompressedByteArray(output, e.getValue().getFirst()); - WritableUtils.writeCompressedByteArray(output, e.getValue().getSecond()); - } - } - break; - default: - throw new IOException("Unrecognized projection type '" + projector.type + "'"); - } + WritableUtils.writeCompressedByteArray(output, projector.tablePrefix); scan.setAttribute(SCAN_PROJECTOR, stream.toByteArray()); } catch (IOException e) { throw new RuntimeException(e); @@ -145,40 +99,8 @@ public static ScanProjector deserializeProjectorFromScan(Scan scan) { ByteArrayInputStream stream = new ByteArrayInputStream(proj); try { DataInputStream input = new DataInputStream(stream); - int t = WritableUtils.readVInt(input); - ProjectionType type = ProjectionType.values()[t]; - if (type == ProjectionType.TABLE) { - byte[] tablePrefix = WritableUtils.readCompressedByteArray(input); - return new ScanProjector(type, tablePrefix, null, null); - } - if (type == ProjectionType.CF) { - int count = WritableUtils.readVInt(input); - Map cfMap = new HashMap(); - for (int i = 0; i < count; i++) { - byte[] cf = WritableUtils.readCompressedByteArray(input); - byte[] renamed = WritableUtils.readCompressedByteArray(input); - cfMap.put(new ImmutableBytesPtr(cf), renamed); - } - return new ScanProjector(type, null, cfMap, null); - } - - int count = WritableUtils.readVInt(input); - Map>> cqMap = - new HashMap>>(); - for (int i = 0; i < count; i++) { - byte[] cf = WritableUtils.readCompressedByteArray(input); - int nQuals = WritableUtils.readVInt(input); - Map> map = - new HashMap>(); - for (int j = 0; j < nQuals; j++) { - byte[] cq = WritableUtils.readCompressedByteArray(input); - byte[] renamedCf = WritableUtils.readCompressedByteArray(input); - byte[] renamedCq = WritableUtils.readCompressedByteArray(input); - map.put(new ImmutableBytesPtr(cq), new Pair(renamedCf, renamedCq)); - } - cqMap.put(new ImmutableBytesPtr(cf), map); - } - return new ScanProjector(type, null, null, cqMap); + byte[] tablePrefix = WritableUtils.readCompressedByteArray(input); + return new ScanProjector(tablePrefix); } catch (IOException e) { throw new RuntimeException(e); } finally { @@ -190,46 +112,18 @@ public static ScanProjector deserializeProjectorFromScan(Scan scan) { } } - public ProjectionType getType() { - return this.type; - } - public byte[] getTablePrefix() { return this.tablePrefix; } - public Map getCfProjectionMap() { - return this.cfProjectionMap; - } - - public Map>> getCqProjectionMap() { - return this.cqProjectionMap; + public KeyValue projectRow(KeyValue kv) { + return KeyValueUtil.newKeyValue(PROJECTED_ROW, getRowFamily(tablePrefix), getRowQualifier(), + kv.getTimestamp(), kv.getBuffer(), kv.getRowOffset(), kv.getRowLength()); } - public KeyValue getProjectedKeyValue(KeyValue kv) { - if (type == ProjectionType.TABLE) { - byte[] cf = getPrefixedColumnFamily(kv.getFamily(), tablePrefix); - return KeyValueUtil.newKeyValue(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength(), - cf, kv.getQualifier(), kv.getTimestamp(), kv.getBuffer(), kv.getValueOffset(), kv.getValueLength()); - } - - if (type == ProjectionType.CF) { - byte[] cf = cfProjectionMap.get(new ImmutableBytesPtr(kv.getFamily())); - if (cf == null) - return kv; - return KeyValueUtil.newKeyValue(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength(), - cf, kv.getQualifier(), kv.getTimestamp(), kv.getBuffer(), kv.getValueOffset(), kv.getValueLength()); - } - - Map> map = cqProjectionMap.get(new ImmutableBytesPtr(kv.getFamily())); - if (map == null) - return kv; - - Pair col = map.get(new ImmutableBytesPtr(kv.getQualifier())); - if (col == null) - return kv; - - return KeyValueUtil.newKeyValue(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength(), - col.getFirst(), col.getSecond(), kv.getTimestamp(), kv.getBuffer(), kv.getValueOffset(), kv.getValueLength()); + public KeyValue projectedKeyValue(KeyValue kv) { + byte[] cf = getPrefixedColumnFamily(kv.getFamily(), tablePrefix); + return KeyValueUtil.newKeyValue(PROJECTED_ROW, cf, kv.getQualifier(), + kv.getTimestamp(), kv.getBuffer(), kv.getValueOffset(), kv.getValueLength()); } } diff --git a/src/main/java/com/salesforce/phoenix/schema/ColumnRef.java b/src/main/java/com/salesforce/phoenix/schema/ColumnRef.java index 7f16e390..b83a896b 100644 --- a/src/main/java/com/salesforce/phoenix/schema/ColumnRef.java +++ b/src/main/java/com/salesforce/phoenix/schema/ColumnRef.java @@ -100,7 +100,7 @@ public boolean equals(Object obj) { public ColumnExpression newColumnExpression() throws SQLException { if (SchemaUtil.isPKColumn(this.getColumn())) { if (disambiguateWithTable) { - return new RowKeyColumnExpression(getColumn(), new RowKeyValueAccessor(this.getTable().getPKColumns(), pkSlotPosition), ScanProjector.getPrefixForTable(tableRef)); + return new KeyValueColumnExpression(getColumn(), ScanProjector.getPrefixForTable(tableRef), new RowKeyValueAccessor(this.getTable().getPKColumns(), pkSlotPosition)); } return new RowKeyColumnExpression(getColumn(), new RowKeyValueAccessor(this.getTable().getPKColumns(), pkSlotPosition)); diff --git a/src/main/java/com/salesforce/phoenix/schema/RowKeyValueAccessor.java b/src/main/java/com/salesforce/phoenix/schema/RowKeyValueAccessor.java index 464f4448..97376f28 100644 --- a/src/main/java/com/salesforce/phoenix/schema/RowKeyValueAccessor.java +++ b/src/main/java/com/salesforce/phoenix/schema/RowKeyValueAccessor.java @@ -29,8 +29,12 @@ import static com.salesforce.phoenix.query.QueryConstants.SEPARATOR_BYTE; -import java.io.*; -import java.util.*; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; @@ -95,6 +99,12 @@ public RowKeyValueAccessor(List data, int index) { this.hasSeparator = !isFixedLength && (datum != data.get(data.size()-1)); } + RowKeyValueAccessor(int[] offsets, boolean isFixedLength, boolean hasSeparator) { + this.offsets = offsets; + this.isFixedLength = isFixedLength; + this.hasSeparator = hasSeparator; + } + private int index = -1; // Only available on client side private int[] offsets; private boolean isFixedLength; @@ -138,7 +148,7 @@ public void readFields(DataInput input) throws IOException { hasSeparator = (length & 0x02) != 0; isFixedLength = (length & 0x01) != 0; length >>= 2; - offsets = ByteUtil.deserializeIntArray(input, length); + offsets = ByteUtil.deserializeVIntArray(input, length); } @Override @@ -147,7 +157,7 @@ public void write(DataOutput output) throws IOException { // (since there's plenty of room) int length = offsets.length << 2; length |= (hasSeparator ? 1 << 1 : 0) | (isFixedLength ? 1 : 0); - ByteUtil.serializeIntArray(output, offsets, length); + ByteUtil.serializeVIntArray(output, offsets, length); } /** @@ -179,12 +189,11 @@ public int getOffset(byte[] keyBuffer, int keyOffset) { * @param keyLength the length of the entire row key * @return the length of the PK column value */ - public int getLength(byte[] keyBuffer, int keyOffset, int keyLength) { + public int getLength(byte[] keyBuffer, int keyOffset, int maxOffset) { if (!hasSeparator) { - return keyLength; + return maxOffset - keyOffset; } int offset = keyOffset; - int maxOffset = keyOffset + keyLength; // FIXME: offset < maxOffset required because HBase passes bogus keys to filter to position scan (HBASE-6562) while (offset < maxOffset && keyBuffer[offset] != SEPARATOR_BYTE) { offset++; diff --git a/src/main/java/com/salesforce/phoenix/schema/tuple/MultiKeyValueTuple.java b/src/main/java/com/salesforce/phoenix/schema/tuple/MultiKeyValueTuple.java index 59a0345c..d1ebbb86 100644 --- a/src/main/java/com/salesforce/phoenix/schema/tuple/MultiKeyValueTuple.java +++ b/src/main/java/com/salesforce/phoenix/schema/tuple/MultiKeyValueTuple.java @@ -31,8 +31,8 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.util.Bytes; +import com.google.common.collect.ImmutableList; import com.salesforce.phoenix.util.KeyValueUtil; @@ -40,14 +40,14 @@ public class MultiKeyValueTuple implements Tuple { private List values; public MultiKeyValueTuple(List values) { - this.values = values; + setKeyValues(values); } public MultiKeyValueTuple() { } public void setKeyValues(List values) { - this.values = values; + this.values = ImmutableList.copyOf(values); } @Override @@ -80,17 +80,4 @@ public int size() { public KeyValue getValue(int index) { return values.get(index); } - - @Override - public boolean getKey(ImmutableBytesWritable ptr, byte[] cfPrefix) { - for (KeyValue kv : values) { - int len = kv.getFamilyLength(); - if (len >= cfPrefix.length - && Bytes.equals(cfPrefix, 0, cfPrefix.length, kv.getBuffer(), kv.getFamilyOffset(), len)) { - ptr.set(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength()); - return true; - } - } - return false; - } } diff --git a/src/main/java/com/salesforce/phoenix/schema/tuple/ResultTuple.java b/src/main/java/com/salesforce/phoenix/schema/tuple/ResultTuple.java index 4ec39cf7..7afd7148 100644 --- a/src/main/java/com/salesforce/phoenix/schema/tuple/ResultTuple.java +++ b/src/main/java/com/salesforce/phoenix/schema/tuple/ResultTuple.java @@ -70,7 +70,24 @@ public KeyValue getValue(byte[] family, byte[] qualifier) { @Override public String toString() { - return result.toString(); + StringBuilder sb = new StringBuilder(); + sb.append("keyvalues="); + if(this.result.isEmpty()) { + sb.append("NONE"); + return sb.toString(); + } + sb.append("{"); + boolean moreThanOne = false; + for(KeyValue kv : this.result.list()) { + if(moreThanOne) { + sb.append(", \n"); + } else { + moreThanOne = true; + } + sb.append(kv.toString()+"/value="+Bytes.toString(kv.getValue())); + } + sb.append("}\n"); + return sb.toString(); } @Override @@ -82,17 +99,4 @@ public int size() { public KeyValue getValue(int index) { return result.raw()[index]; } - - @Override - public boolean getKey(ImmutableBytesWritable ptr, byte[] cfPrefix) { - for (KeyValue kv : result.raw()) { - int len = kv.getFamilyLength(); - if (len >= cfPrefix.length - && Bytes.equals(cfPrefix, 0, cfPrefix.length, kv.getBuffer(), kv.getFamilyOffset(), len)) { - ptr.set(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength()); - return true; - } - } - return false; - } } diff --git a/src/main/java/com/salesforce/phoenix/schema/tuple/SingleKeyValueTuple.java b/src/main/java/com/salesforce/phoenix/schema/tuple/SingleKeyValueTuple.java index ccf7a94a..6dc49a70 100644 --- a/src/main/java/com/salesforce/phoenix/schema/tuple/SingleKeyValueTuple.java +++ b/src/main/java/com/salesforce/phoenix/schema/tuple/SingleKeyValueTuple.java @@ -107,15 +107,4 @@ public KeyValue getValue(int index) { } return keyValue; } - - @Override - public boolean getKey(ImmutableBytesWritable ptr, byte[] cfPrefix) { - int len = keyValue.getFamilyLength(); - if (len >= cfPrefix.length - && Bytes.equals(cfPrefix, 0, cfPrefix.length, keyValue.getBuffer(), keyValue.getFamilyOffset(), len)) { - ptr.set(keyValue.getBuffer(), keyValue.getKeyOffset(), keyValue.getKeyLength()); - return true; - } - return false; - } } diff --git a/src/main/java/com/salesforce/phoenix/schema/tuple/Tuple.java b/src/main/java/com/salesforce/phoenix/schema/tuple/Tuple.java index 352486c6..6858d5b4 100644 --- a/src/main/java/com/salesforce/phoenix/schema/tuple/Tuple.java +++ b/src/main/java/com/salesforce/phoenix/schema/tuple/Tuple.java @@ -60,18 +60,6 @@ public interface Tuple { */ public void getKey(ImmutableBytesWritable ptr); - /** - * Get the row key for the Tuple where the row key is contained - * in a KeyValue having the specified column family prefix. - * This is used for searching a joined result for a row key from - * a specific table. - * @param ptr the bytes pointer that will be updated to point to - * the key buffer. - * @param cfPrefix the column family prefix. - * @return true if the specified KeyValue is found. - */ - public boolean getKey(ImmutableBytesWritable ptr, byte[] cfPrefix); - /** * Get the KeyValue at the given index. * @param index the zero-based KeyValue index between 0 and {@link #size()} exclusive diff --git a/src/test/java/com/salesforce/phoenix/end2end/HashJoinTest.java b/src/test/java/com/salesforce/phoenix/end2end/HashJoinTest.java index 2a8ca03b..513111b7 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/HashJoinTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/HashJoinTest.java @@ -27,8 +27,15 @@ ******************************************************************************/ package com.salesforce.phoenix.end2end; -import static com.salesforce.phoenix.util.TestUtil.*; -import static org.junit.Assert.*; +import static com.salesforce.phoenix.util.TestUtil.JOIN_CUSTOMER_TABLE; +import static com.salesforce.phoenix.util.TestUtil.JOIN_ITEM_TABLE; +import static com.salesforce.phoenix.util.TestUtil.JOIN_SUPPLIER_TABLE; +import static com.salesforce.phoenix.util.TestUtil.PHOENIX_JDBC_URL; +import static com.salesforce.phoenix.util.TestUtil.TEST_PROPERTIES; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import java.sql.Connection; import java.sql.DriverManager; @@ -38,17 +45,14 @@ import org.junit.Test; -import com.salesforce.phoenix.util.PhoenixRuntime; - -public abstract class HashJoinTest extends BaseClientMangedTimeTest { +public class HashJoinTest extends BaseClientMangedTimeTest { - private void initMetaInfoTableValues(Long ts) throws Exception { + private void initMetaInfoTableValues() throws Exception { ensureTableCreated(getUrl(), JOIN_CUSTOMER_TABLE); ensureTableCreated(getUrl(), JOIN_ITEM_TABLE); ensureTableCreated(getUrl(), JOIN_SUPPLIER_TABLE); - Properties props = new Properties(); - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, ts.toString()); + Properties props = new Properties(TEST_PROPERTIES); Connection conn = DriverManager.getConnection(getUrl(), props); try { // Insert into customer table @@ -103,7 +107,7 @@ private void initMetaInfoTableValues(Long ts) throws Exception { " PRICE, " + " SUPPLIER_ID, " + " DESCRIPTION) " + - "values (?, ?, ?, ?)"); + "values (?, ?, ?, ?, ?)"); stmt.setString(1, "0000000001"); stmt.setString(2, "T1"); stmt.setInt(3, 100); @@ -205,11 +209,9 @@ private void initMetaInfoTableValues(Long ts) throws Exception { @Test public void testInnerJoin() throws Exception { - long ts = nextTimestamp(); - initMetaInfoTableValues(ts); + initMetaInfoTableValues(); String query = "SELECT item.item_id, item.name, supp.supplier_id, supp.name FROM " + JOIN_ITEM_TABLE + " item INNER JOIN " + JOIN_SUPPLIER_TABLE + " supp ON item.supplier_id = supp.supplier_id"; Properties props = new Properties(TEST_PROPERTIES); - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(PHOENIX_JDBC_URL, props); try { PreparedStatement statement = conn.prepareStatement(query); @@ -253,11 +255,9 @@ public void testInnerJoin() throws Exception { @Test public void testLeftJoin() throws Exception { - long ts = nextTimestamp(); - initMetaInfoTableValues(ts); + initMetaInfoTableValues(); String query = "SELECT item.item_id, item.name, supp.supplier_id, supp.name FROM " + JOIN_ITEM_TABLE + " item LEFT JOIN " + JOIN_SUPPLIER_TABLE + " supp ON item.supplier_id = supp.supplier_id"; Properties props = new Properties(TEST_PROPERTIES); - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(PHOENIX_JDBC_URL, props); try { PreparedStatement statement = conn.prepareStatement(query); diff --git a/src/test/java/com/salesforce/phoenix/end2end/OrderByTest.java b/src/test/java/com/salesforce/phoenix/end2end/OrderByTest.java index b591461c..4fd024b5 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/OrderByTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/OrderByTest.java @@ -37,7 +37,7 @@ import com.salesforce.phoenix.util.PhoenixRuntime; -public abstract class OrderByTest extends BaseClientMangedTimeTest { +public class OrderByTest extends BaseClientMangedTimeTest { @Test public void testMultiOrderByExpr() throws Exception { diff --git a/src/test/java/com/salesforce/phoenix/query/BaseTest.java b/src/test/java/com/salesforce/phoenix/query/BaseTest.java index b3b53dae..f50d77d7 100644 --- a/src/test/java/com/salesforce/phoenix/query/BaseTest.java +++ b/src/test/java/com/salesforce/phoenix/query/BaseTest.java @@ -243,31 +243,27 @@ public abstract class BaseTest { builder.put("SumDoubleTest","create table SumDoubleTest" + " (id varchar not null primary key, d DOUBLE, f FLOAT, ud UNSIGNED_DOUBLE, uf UNSIGNED_FLOAT, i integer, de decimal)"); builder.put(JOIN_ORDER_TABLE, "create table " + JOIN_ORDER_TABLE + - " (order_id char(15) not null, " + + " (order_id char(15) not null primary key, " + " customer_id char(10) not null, " + " item_id char(10) not null, " + " quantity integer not null, " + - " date date not null " + - " CONSTRAINT pk PRIMARY KEY (order_id))"); + " date date not null)"); builder.put(JOIN_CUSTOMER_TABLE, "create table " + JOIN_CUSTOMER_TABLE + - " (customer_id char(10) not null, " + + " (customer_id char(10) not null primary key, " + " name varchar not null, " + " phone char(12), " + - " address varchar " + - " CONSTRAINT pk PRIMARY KEY (customer_id))"); + " address varchar)"); builder.put(JOIN_ITEM_TABLE, "create table " + JOIN_ITEM_TABLE + - " (item_id char(10) not null, " + + " (item_id char(10) not null primary key, " + " name varchar not null, " + " price integer not null, " + " supplier_id char(10) not null, " + - " description varchar " + - " CONSTRAINT pk PRIMARY KEY (item_id))"); + " description varchar)"); builder.put(JOIN_SUPPLIER_TABLE, "create table " + JOIN_SUPPLIER_TABLE + - " (supplier_id char(10) not null, " + + " (supplier_id char(10) not null primary key, " + " name varchar not null, " + " phone char(12), " + - " address varchar " + - " CONSTRAINT pk PRIMARY KEY (supplier_id))"); + " address varchar)"); tableDDLMap = builder.build(); } From 6a3bb55cd460b5d45ca114cdd07dee18cbeebbb8 Mon Sep 17 00:00:00 2001 From: maryannxue Date: Wed, 2 Oct 2013 17:09:56 -0400 Subject: [PATCH 099/102] bug fix --- .../java/com/salesforce/phoenix/compile/FromCompiler.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/salesforce/phoenix/compile/FromCompiler.java b/src/main/java/com/salesforce/phoenix/compile/FromCompiler.java index 0db8edc6..ca7aa539 100644 --- a/src/main/java/com/salesforce/phoenix/compile/FromCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/FromCompiler.java @@ -297,7 +297,11 @@ public void visit(NamedTableNode namedTableNode) throws SQLException { tableMap.put(alias, tableRef); } - tableMap.put( theTable.getName().getString(), tableRef); + String name = theTable.getName().getString(); + //avoid having one name mapped to two identical TableRef. + if (alias == null || !alias.equals(name)) { + tableMap.put(name, tableRef); + } tables.add(tableRef); } From 1c98cf3e2481c14e8931d292d0d8cbbc05861dd5 Mon Sep 17 00:00:00 2001 From: maryannxue Date: Wed, 2 Oct 2013 18:10:50 -0400 Subject: [PATCH 100/102] fix OnNodeVisitor --- .../phoenix/compile/JoinCompiler.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java index dfdd1214..42f8d05f 100644 --- a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java @@ -46,10 +46,12 @@ import com.salesforce.phoenix.parse.AliasedNode; import com.salesforce.phoenix.parse.AndParseNode; import com.salesforce.phoenix.parse.CaseParseNode; +import com.salesforce.phoenix.parse.CastParseNode; import com.salesforce.phoenix.parse.ColumnParseNode; import com.salesforce.phoenix.parse.ComparisonParseNode; import com.salesforce.phoenix.parse.ConcreteTableNode; import com.salesforce.phoenix.parse.EqualParseNode; +import com.salesforce.phoenix.parse.FunctionParseNode; import com.salesforce.phoenix.parse.InListParseNode; import com.salesforce.phoenix.parse.IsNullParseNode; import com.salesforce.phoenix.parse.JoinTableNode; @@ -281,6 +283,24 @@ public Void visitLeave(IsNullParseNode node, List l) throws SQLException { return leaveBooleanNode(node, l); } + + @Override + public Void visitLeave(FunctionParseNode node, List l) + throws SQLException { + return leaveBooleanNode(node, l); + } + + @Override + public Void visitLeave(CaseParseNode node, List l) + throws SQLException { + return leaveBooleanNode(node, l); + } + + @Override + public Void visitLeave(CastParseNode node, List l) + throws SQLException { + return leaveBooleanNode(node, l); + } } } @@ -515,6 +535,24 @@ public Void visitLeave(IsNullParseNode node, List l) throws SQLException { return leaveNonEqBooleanNode(node, l); } + + @Override + public Void visitLeave(FunctionParseNode node, List l) + throws SQLException { + return leaveNonEqBooleanNode(node, l); + } + + @Override + public Void visitLeave(CaseParseNode node, List l) + throws SQLException { + return leaveNonEqBooleanNode(node, l); + } + + @Override + public Void visitLeave(CastParseNode node, List l) + throws SQLException { + return leaveNonEqBooleanNode(node, l); + } } } From 1495a9f3303805a9092e2a45ee735689477f3f68 Mon Sep 17 00:00:00 2001 From: maryannxue Date: Wed, 2 Oct 2013 21:40:22 -0400 Subject: [PATCH 101/102] bug fix in JoinCompiler --- src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java index 42f8d05f..f843f886 100644 --- a/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java +++ b/src/main/java/com/salesforce/phoenix/compile/JoinCompiler.java @@ -306,7 +306,7 @@ public Void visitLeave(CastParseNode node, List l) public static JoinSpec getSubJoinSpec(JoinSpec join) { return new JoinSpec(join.mainTable, join.select, join.preFilters, join.postFilters, - join.joinTables.subList(0, join.joinTables.size() - 2), join.columnRefs); + join.joinTables.subList(0, join.joinTables.size() - 1), join.columnRefs); } public static class JoinTable { From 7719a93fbf56a0a169760d8faa35ea99ef766afc Mon Sep 17 00:00:00 2001 From: maryannxue Date: Wed, 2 Oct 2013 21:42:09 -0400 Subject: [PATCH 102/102] Add testRightJoin for HashJoinTest --- .../phoenix/end2end/HashJoinTest.java | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/test/java/com/salesforce/phoenix/end2end/HashJoinTest.java b/src/test/java/com/salesforce/phoenix/end2end/HashJoinTest.java index 513111b7..ebcfdc77 100644 --- a/src/test/java/com/salesforce/phoenix/end2end/HashJoinTest.java +++ b/src/test/java/com/salesforce/phoenix/end2end/HashJoinTest.java @@ -303,5 +303,56 @@ public void testLeftJoin() throws Exception { conn.close(); } } + + @Test + public void testRightJoin() throws Exception { + initMetaInfoTableValues(); + String query = "SELECT item.item_id, item.name, supp.supplier_id, supp.name FROM " + JOIN_SUPPLIER_TABLE + " supp RIGHT JOIN " + JOIN_ITEM_TABLE + " item ON item.supplier_id = supp.supplier_id"; + Properties props = new Properties(TEST_PROPERTIES); + Connection conn = DriverManager.getConnection(PHOENIX_JDBC_URL, props); + try { + PreparedStatement statement = conn.prepareStatement(query); + ResultSet rs = statement.executeQuery(); + assertTrue (rs.next()); + assertEquals(rs.getString(1), "0000000001"); + assertEquals(rs.getString(2), "T1"); + assertEquals(rs.getString(3), "0000000001"); + assertEquals(rs.getString(4), "S1"); + assertTrue (rs.next()); + assertEquals(rs.getString(1), "0000000002"); + assertEquals(rs.getString(2), "T2"); + assertEquals(rs.getString(3), "0000000001"); + assertEquals(rs.getString(4), "S1"); + assertTrue (rs.next()); + assertEquals(rs.getString(1), "0000000003"); + assertEquals(rs.getString(2), "T3"); + assertEquals(rs.getString(3), "0000000002"); + assertEquals(rs.getString(4), "S2"); + assertTrue (rs.next()); + assertEquals(rs.getString(1), "0000000004"); + assertEquals(rs.getString(2), "T4"); + assertEquals(rs.getString(3), "0000000002"); + assertEquals(rs.getString(4), "S2"); + assertTrue (rs.next()); + assertEquals(rs.getString(1), "0000000005"); + assertEquals(rs.getString(2), "T5"); + assertEquals(rs.getString(3), "0000000005"); + assertEquals(rs.getString(4), "S5"); + assertTrue (rs.next()); + assertEquals(rs.getString(1), "0000000006"); + assertEquals(rs.getString(2), "T6"); + assertEquals(rs.getString(3), "0000000006"); + assertEquals(rs.getString(4), "S6"); + assertTrue (rs.next()); + assertEquals(rs.getString(1), "invalid001"); + assertEquals(rs.getString(2), "INVALID-1"); + assertNull(rs.getString(3)); + assertNull(rs.getString(4)); + + assertFalse(rs.next()); + } finally { + conn.close(); + } + } }