diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java index 4de8ecc88c86..c1f98edd75ab 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java @@ -30,6 +30,7 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; @@ -53,6 +54,8 @@ import org.apache.hadoop.hbase.ClusterMetrics.Option; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.UnknownRegionException; import org.apache.hadoop.hbase.client.Admin; @@ -60,10 +63,16 @@ import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.DoNotRetryRegionException; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.master.RackManager; +import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.assignment.AssignmentManager; import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.rsgroup.RSGroupInfo; +import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; +import org.apache.hadoop.hbase.zookeeper.ZKWatcher; +import org.apache.hadoop.hbase.zookeeper.ZNodePaths; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -96,6 +105,7 @@ public class RegionMover extends AbstractHBaseTool implements Closeable { private boolean ack = true; private int maxthreads = 1; private int timeout; + private List isolateRegionIdArray; private String loadUnload; private String hostname; private String filename; @@ -112,6 +122,7 @@ private RegionMover(RegionMoverBuilder builder) throws IOException { this.excludeFile = builder.excludeFile; this.designatedFile = builder.designatedFile; this.maxthreads = builder.maxthreads; + this.isolateRegionIdArray = builder.isolateRegionIdArray; this.ack = builder.ack; this.port = builder.port; this.timeout = builder.timeout; @@ -156,6 +167,7 @@ public static class RegionMoverBuilder { private boolean ack = true; private int maxthreads = 1; private int timeout = Integer.MAX_VALUE; + private List isolateRegionIdArray = new ArrayList<>(); private String hostname; private String filename; private String excludeFile = null; @@ -216,6 +228,14 @@ public RegionMoverBuilder maxthreads(int threads) { return this; } + /** + * Set the region ID to isolate on the region server. + */ + public RegionMoverBuilder isolateRegionIdArray(List isolateRegionIdArray) { + this.isolateRegionIdArray = isolateRegionIdArray; + return this; + } + /** * Path of file containing hostnames to be excluded during region movement. Exclude file should * have 'host:port' per line. Port is mandatory here as we can have many RS running on a single @@ -409,6 +429,25 @@ public boolean unloadFromRack() } private boolean unloadRegions(boolean unloadFromRack) + throws ExecutionException, InterruptedException, TimeoutException { + return unloadRegions(unloadFromRack, null); + } + + /** + * Isolated regions specified in {@link #isolateRegionIdArray} on {@link #hostname} in ack Mode + * and Unload regions from given {@link #hostname} using ack/noAck mode and {@link #maxthreads}. + * In noAck mode we do not make sure that region is successfully online on the target region + * server,hence it is the best effort. We do not unload regions to hostnames given in + * {@link #excludeFile}. If designatedFile is present with some contents, we will unload regions + * to hostnames provided in {@link #designatedFile} + * @return true if region isolation succeeded, false otherwise + */ + public boolean isolateRegions() + throws ExecutionException, InterruptedException, TimeoutException { + return unloadRegions(false, isolateRegionIdArray); + } + + private boolean unloadRegions(boolean unloadFromRack, List isolateRegionIdArray) throws InterruptedException, ExecutionException, TimeoutException { deleteFile(this.filename); ExecutorService unloadPool = Executors.newFixedThreadPool(1); @@ -466,7 +505,7 @@ private boolean unloadRegions(boolean unloadFromRack) } else { LOG.info("Available servers {}", regionServers); } - unloadRegions(server, regionServers, movedRegions); + unloadRegions(server, regionServers, movedRegions, isolateRegionIdArray); } catch (Exception e) { LOG.error("Error while unloading regions ", e); return false; @@ -497,9 +536,111 @@ Collection filterRSGroupServers(RSGroupInfo rsgroup, } private void unloadRegions(ServerName server, List regionServers, - List movedRegions) throws Exception { + List movedRegions, List isolateRegionIdArray) throws Exception { while (true) { + List isolateRegionInfoList = Collections.synchronizedList(new ArrayList<>()); + RegionInfo isolateRegionInfo = null; + if (isolateRegionIdArray != null && !isolateRegionIdArray.isEmpty()) { + // Region will be moved to target region server with Ack mode. + final ExecutorService isolateRegionPool = Executors.newFixedThreadPool(maxthreads); + List> isolateRegionTaskList = new ArrayList<>(); + List recentlyIsolatedRegion = Collections.synchronizedList(new ArrayList<>()); + boolean allRegionOpsSuccessful = true; + boolean isMetaIsolated = false; + RegionInfo metaRegionInfo = RegionInfoBuilder.FIRST_META_REGIONINFO; + List hRegionLocationRegionIsolation = + Collections.synchronizedList(new ArrayList<>()); + for (String isolateRegionId : isolateRegionIdArray) { + if (isolateRegionId.equalsIgnoreCase(metaRegionInfo.getEncodedName())) { + isMetaIsolated = true; + continue; + } + Result result = MetaTableAccessor.scanByRegionEncodedName(conn, isolateRegionId); + HRegionLocation hRegionLocation = + MetaTableAccessor.getRegionLocation(conn, result.getRow()); + if (hRegionLocation != null) { + hRegionLocationRegionIsolation.add(hRegionLocation); + } else { + LOG.error("Region " + isolateRegionId + " doesn't exists/can't fetch from" + + " meta...Quitting now"); + // We only move the regions if all the regions were found. + allRegionOpsSuccessful = false; + break; + } + } + + if (!allRegionOpsSuccessful) { + break; + } + // If hbase:meta region was isolated, then it needs to be part of isolateRegionInfoList. + if (isMetaIsolated) { + ZKWatcher zkWatcher = new ZKWatcher(conf, null, null); + List result = new ArrayList<>(); + for (String znode : zkWatcher.getMetaReplicaNodes()) { + String path = ZNodePaths.joinZNode(zkWatcher.getZNodePaths().baseZNode, znode); + int replicaId = zkWatcher.getZNodePaths().getMetaReplicaIdFromPath(path); + RegionState state = MetaTableLocator.getMetaRegionState(zkWatcher, replicaId); + result.add(new HRegionLocation(state.getRegion(), state.getServerName())); + } + ServerName metaSeverName = result.get(0).getServerName(); + // For isolating hbase:meta, it should move explicitly in Ack mode, + // hence the forceMoveRegionByAck = true. + if (!metaSeverName.equals(server)) { + LOG.info("Region of hbase:meta " + metaRegionInfo.getEncodedName() + " is on server " + + metaSeverName + " moving to " + server); + submitRegionMovesWhileUnloading(metaSeverName, Collections.singletonList(server), + movedRegions, Collections.singletonList(metaRegionInfo), true); + } else { + LOG.info("Region of hbase:meta " + metaRegionInfo.getEncodedName() + " already exists" + + " on server : " + server); + } + isolateRegionInfoList.add(RegionInfoBuilder.FIRST_META_REGIONINFO); + } + + if (!hRegionLocationRegionIsolation.isEmpty()) { + for (HRegionLocation hRegionLocation : hRegionLocationRegionIsolation) { + isolateRegionInfo = hRegionLocation.getRegion(); + isolateRegionInfoList.add(isolateRegionInfo); + if (hRegionLocation.getServerName() == server) { + LOG.info("Region " + hRegionLocation.getRegion().getEncodedName() + " already exists" + + " on server : " + server.getHostname()); + } else { + Future isolateRegionTask = + isolateRegionPool.submit(new MoveWithAck(conn, isolateRegionInfo, + hRegionLocation.getServerName(), server, recentlyIsolatedRegion)); + isolateRegionTaskList.add(isolateRegionTask); + } + } + } + + if (!isolateRegionTaskList.isEmpty()) { + isolateRegionPool.shutdown(); + // Now that we have fetched all the region's regionInfo, we can move them. + waitMoveTasksToFinish(isolateRegionPool, isolateRegionTaskList, + admin.getConfiguration().getLong(MOVE_WAIT_MAX_KEY, DEFAULT_MOVE_WAIT_MAX)); + + Set currentRegionsOnTheServer = new HashSet<>(admin.getRegions(server)); + if (!currentRegionsOnTheServer.containsAll(isolateRegionInfoList)) { + // If all the regions are not online on the target server, + // we don't put RS in decommission mode and exit from here. + LOG.error("One of the Region move failed OR stuck in transition...Quitting now"); + break; + } + } else { + LOG.info("All regions already exists on server : " + server.getHostname()); + } + // Once region has been moved to target RS, put the target RS into decommission mode, + // so master doesn't assign new region to the target RS while we unload the target RS. + // Also pass 'offload' flag as false since we don't want master to offload the target RS. + List listOfServer = new ArrayList<>(); + listOfServer.add(server); + LOG.info("Putting server : " + server.getHostname() + " in decommission/draining mode"); + admin.decommissionRegionServers(listOfServer, false); + } List regionsToMove = admin.getRegions(server); + // Remove all the regions from the online Region list, that we just isolated. + // This will also include hbase:meta if it was isolated. + regionsToMove.removeAll(isolateRegionInfoList); regionsToMove.removeAll(movedRegions); if (regionsToMove.isEmpty()) { LOG.info("No Regions to move....Quitting now"); @@ -511,21 +652,25 @@ private void unloadRegions(ServerName server, List regionServers, Optional metaRegion = getMetaRegionInfoIfToBeMoved(regionsToMove); if (metaRegion.isPresent()) { RegionInfo meta = metaRegion.get(); + // hbase:meta should move explicitly in Ack mode. submitRegionMovesWhileUnloading(server, regionServers, movedRegions, - Collections.singletonList(meta)); + Collections.singletonList(meta), true); regionsToMove.remove(meta); } - submitRegionMovesWhileUnloading(server, regionServers, movedRegions, regionsToMove); + submitRegionMovesWhileUnloading(server, regionServers, movedRegions, regionsToMove, false); } } private void submitRegionMovesWhileUnloading(ServerName server, List regionServers, - List movedRegions, List regionsToMove) throws Exception { + List movedRegions, List regionsToMove, boolean forceMoveRegionByAck) + throws Exception { final ExecutorService moveRegionsPool = Executors.newFixedThreadPool(this.maxthreads); List> taskList = new ArrayList<>(); int serverIndex = 0; for (RegionInfo regionToMove : regionsToMove) { - if (ack) { + // To move/isolate hbase:meta on a server, it should happen explicitly by Ack mode, hence the + // forceMoveRegionByAck = true. + if (ack || forceMoveRegionByAck) { Future task = moveRegionsPool.submit(new MoveWithAck(conn, regionToMove, server, regionServers.get(serverIndex), movedRegions)); taskList.add(task); @@ -771,9 +916,17 @@ private ServerName stripServer(List regionServers, String hostname, @Override protected void addOptions() { this.addRequiredOptWithArg("r", "regionserverhost", "region server |"); - this.addRequiredOptWithArg("o", "operation", "Expected: load/unload/unload_from_rack"); + this.addRequiredOptWithArg("o", "operation", + "Expected: load/unload/unload_from_rack/isolate_regions"); this.addOptWithArg("m", "maxthreads", "Define the maximum number of threads to use to unload and reload the regions"); + this.addOptWithArg("i", "isolateRegionIds", + "Comma separated list of Region IDs hash to isolate on a RegionServer and put region server" + + " in draining mode. This option should only be used with '-o isolate_regions'." + + " By putting region server in decommission/draining mode, master can't assign any" + + " new region on this server. If one or more regions are not found OR failed to isolate" + + " successfully, utility will exist without putting RS in draining/decommission mode." + + " Ex. --isolateRegionIds id1,id2,id3 OR -i id1,id2,id3"); this.addOptWithArg("x", "excludefile", "File with per line to exclude as unload targets; default excludes only " + "target host; useful for rack decommisioning."); @@ -795,9 +948,14 @@ protected void addOptions() { protected void processOptions(CommandLine cmd) { String hostname = cmd.getOptionValue("r"); rmbuilder = new RegionMoverBuilder(hostname); + this.loadUnload = cmd.getOptionValue("o").toLowerCase(Locale.ROOT); if (cmd.hasOption('m')) { rmbuilder.maxthreads(Integer.parseInt(cmd.getOptionValue('m'))); } + if (this.loadUnload.equals("isolate_regions") && cmd.hasOption("isolateRegionIds")) { + rmbuilder + .isolateRegionIdArray(Arrays.asList(cmd.getOptionValue("isolateRegionIds").split(","))); + } if (cmd.hasOption('n')) { rmbuilder.ack(false); } @@ -813,7 +971,6 @@ protected void processOptions(CommandLine cmd) { if (cmd.hasOption('t')) { rmbuilder.timeout(Integer.parseInt(cmd.getOptionValue('t'))); } - this.loadUnload = cmd.getOptionValue("o").toLowerCase(Locale.ROOT); } @Override @@ -826,6 +983,15 @@ protected int doWork() throws Exception { success = rm.unload(); } else if (loadUnload.equalsIgnoreCase("unload_from_rack")) { success = rm.unloadFromRack(); + } else if (loadUnload.equalsIgnoreCase("isolate_regions")) { + if (rm.isolateRegionIdArray != null && !rm.isolateRegionIdArray.isEmpty()) { + success = rm.isolateRegions(); + } else { + LOG.error("Missing -i/--isolate_regions option with '-o isolate_regions' option"); + LOG.error("Use -h or --help for usage instructions"); + printUsage(); + success = false; + } } else { printUsage(); success = false; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover2.java index 2f145ad3d023..ec8c44592e94 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMover2.java @@ -23,18 +23,25 @@ import java.util.stream.Collectors; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.SingleProcessHBaseCluster; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; +import org.apache.hadoop.hbase.zookeeper.ZKWatcher; +import org.apache.hadoop.hbase.zookeeper.ZNodePaths; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -210,4 +217,174 @@ public void testFailedRegionMove() throws Exception { } } + public void loadDummyDataInTable(TableName tableName) throws Exception { + Admin admin = TEST_UTIL.getAdmin(); + Table table = TEST_UTIL.getConnection().getTable(tableName); + List puts = new ArrayList<>(); + for (int i = 0; i < 1000; i++) { + puts.add(new Put(Bytes.toBytes("rowkey_" + i)).addColumn(Bytes.toBytes("fam1"), + Bytes.toBytes("q1"), Bytes.toBytes("val_" + i))); + } + table.put(puts); + admin.flush(tableName); + } + + @Test + public void testIsolateSingleRegionOnTheSameServer() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + loadDummyDataInTable(tableName); + ServerName sourceServerName = findSourceServerName(tableName); + // Isolating 1 region on the same region server. + regionIsolationOperation(sourceServerName, sourceServerName, 1, false); + } + + @Test + public void testIsolateSingleRegionOnTheDifferentServer() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + loadDummyDataInTable(tableName); + ServerName sourceServerName = findSourceServerName(tableName); + ServerName destinationServerName = findDestinationServerName(sourceServerName); + // Isolating 1 region on the different region server. + regionIsolationOperation(sourceServerName, destinationServerName, 1, false); + } + + @Test + public void testIsolateMultipleRegionsOnTheSameServer() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + loadDummyDataInTable(tableName); + ServerName sourceServerName = findSourceServerName(tableName); + // Isolating 2 regions on the same region server. + regionIsolationOperation(sourceServerName, sourceServerName, 2, false); + } + + @Test + public void testIsolateMultipleRegionsOnTheDifferentServer() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + loadDummyDataInTable(tableName); + // Isolating 2 regions on the different region server. + ServerName sourceServerName = findSourceServerName(tableName); + ServerName destinationServerName = findDestinationServerName(sourceServerName); + regionIsolationOperation(sourceServerName, destinationServerName, 2, false); + } + + @Test + public void testIsolateMetaOnTheSameSever() throws Exception { + ServerName metaServerSource = findMetaRSLocation(); + regionIsolationOperation(metaServerSource, metaServerSource, 1, true); + } + + @Test + public void testIsolateMetaOnTheDifferentServer() throws Exception { + ServerName metaServerSource = findMetaRSLocation(); + ServerName metaServerDestination = findDestinationServerName(metaServerSource); + regionIsolationOperation(metaServerSource, metaServerDestination, 1, true); + } + + @Test + public void testIsolateMetaAndRandomRegionOnTheMetaServer() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + loadDummyDataInTable(tableName); + ServerName metaServerSource = findMetaRSLocation(); + ServerName randomSeverRegion = findSourceServerName(tableName); + regionIsolationOperation(randomSeverRegion, metaServerSource, 2, true); + } + + @Test + public void testIsolateMetaAndRandomRegionOnTheRandomServer() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + loadDummyDataInTable(tableName); + ServerName randomSeverRegion = findSourceServerName(tableName); + regionIsolationOperation(randomSeverRegion, randomSeverRegion, 2, true); + } + + public ServerName findMetaRSLocation() throws Exception { + ZKWatcher zkWatcher = new ZKWatcher(TEST_UTIL.getConfiguration(), null, null); + List result = new ArrayList<>(); + for (String znode : zkWatcher.getMetaReplicaNodes()) { + String path = ZNodePaths.joinZNode(zkWatcher.getZNodePaths().baseZNode, znode); + int replicaId = zkWatcher.getZNodePaths().getMetaReplicaIdFromPath(path); + RegionState state = MetaTableLocator.getMetaRegionState(zkWatcher, replicaId); + result.add(new HRegionLocation(state.getRegion(), state.getServerName())); + } + return result.get(0).getServerName(); + } + + public ServerName findSourceServerName(TableName tableName) throws Exception { + SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + int numOfRS = cluster.getNumLiveRegionServers(); + ServerName sourceServer = null; + for (int i = 0; i < numOfRS; i++) { + HRegionServer regionServer = cluster.getRegionServer(i); + List hRegions = regionServer.getRegions().stream() + .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName)) + .collect(Collectors.toList()); + if (hRegions.size() >= 2) { + sourceServer = regionServer.getServerName(); + break; + } + } + if (sourceServer == null) { + throw new Exception( + "This shouln't happen, No RS found with more than 2 regions of table : " + tableName); + } + return sourceServer; + } + + public ServerName findDestinationServerName(ServerName sourceServerName) throws Exception { + SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + ServerName destinationServerName = null; + int numOfRS = cluster.getNumLiveRegionServers(); + for (int i = 0; i < numOfRS; i++) { + destinationServerName = cluster.getRegionServer(i).getServerName(); + if (!destinationServerName.equals(sourceServerName)) { + break; + } + } + if (destinationServerName == null) { + throw new Exception("This shouldn't happen, No RS found which is different than source RS"); + } + return destinationServerName; + } + + public void regionIsolationOperation(ServerName sourceServerName, + ServerName destinationServerName, int numRegionsToIsolate, boolean isolateMetaAlso) + throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + Admin admin = TEST_UTIL.getAdmin(); + HRegionServer sourceRS = cluster.getRegionServer(sourceServerName); + List hRegions = sourceRS.getRegions().stream() + .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName)) + .collect(Collectors.toList()); + List listOfRegionIDsToIsolate = new ArrayList<>(); + for (int i = 0; i < numRegionsToIsolate; i++) { + listOfRegionIDsToIsolate.add(hRegions.get(i).getRegionInfo().getEncodedName()); + } + + if (isolateMetaAlso) { + listOfRegionIDsToIsolate.remove(0); + listOfRegionIDsToIsolate.add(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); + } + + HRegionServer destinationRS = cluster.getRegionServer(destinationServerName); + String destinationRSName = destinationRS.getServerName().getAddress().toString(); + RegionMover.RegionMoverBuilder rmBuilder = + new RegionMover.RegionMoverBuilder(destinationRSName, TEST_UTIL.getConfiguration()).ack(true) + .maxthreads(8).isolateRegionIdArray(listOfRegionIDsToIsolate); + try (RegionMover rm = rmBuilder.build()) { + LOG.debug("Unloading {} except regions : {}", destinationRS.getServerName(), + listOfRegionIDsToIsolate); + rm.isolateRegions(); + Assert.assertEquals(numRegionsToIsolate, destinationRS.getNumberOfOnlineRegions()); + List onlineRegions = destinationRS.getRegions(); + for (int i = 0; i < numRegionsToIsolate; i++) { + Assert.assertTrue( + listOfRegionIDsToIsolate.contains(onlineRegions.get(i).getRegionInfo().getEncodedName())); + } + LOG.debug("Successfully Isolated " + listOfRegionIDsToIsolate.size() + " regions : " + + listOfRegionIDsToIsolate + " on " + destinationRS.getServerName()); + } finally { + admin.recommissionRegionServer(destinationRS.getServerName(), null); + } + } }