diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index 4e6fe9874aad..b919ac9c5e56 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -154,6 +154,13 @@ public enum OperationStatusCode { /** Default value for the balancer period */ public static final int DEFAULT_HBASE_BALANCER_PERIOD = 300000; + /** Config for the oldWALs directory size updater period */ + public static final String HBASE_OLDWAL_DIR_SIZE_UPDATER_PERIOD = + "hbase.master.oldwals.dir.updater.period"; + + /** Default value for the oldWALs directory size updater period */ + public static final int DEFAULT_HBASE_OLDWAL_DIR_SIZE_UPDATER_PERIOD = 300000; + /** * Config key for enable/disable automatically separate child regions to different region servers * in the procedure of split regions. One child will be kept to the server where parent region is diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index 3c293d9c6cd4..95dea3ec4393 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -617,6 +617,12 @@ possible configurations would overwhelm and obscure the important. Period at which the region balancer runs in the Master, in milliseconds. + + hbase.master.oldwals.dir.updater.period + 300000 + Period at which the oldWALs directory size calculator/updater will run in the + Master, in milliseconds. + hbase.regions.slop 0.2 diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java index 5952bdc4d8e8..7bbd57e98b47 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java @@ -66,6 +66,7 @@ public interface MetricsMasterSource extends BaseSource { String MERGE_PLAN_COUNT_NAME = "mergePlanCount"; String CLUSTER_REQUESTS_NAME = "clusterRequests"; + String OLD_WAL_DIR_SIZE_NAME = "oldWALsDirSize"; String MASTER_ACTIVE_TIME_DESC = "Master Active Time"; String MASTER_START_TIME_DESC = "Master Start Time"; String MASTER_FINISHED_INITIALIZATION_TIME_DESC = @@ -85,6 +86,7 @@ public interface MetricsMasterSource extends BaseSource { String MERGE_PLAN_COUNT_DESC = "Number of Region Merge Plans executed"; String SERVER_CRASH_METRIC_PREFIX = "serverCrash"; + String OLD_WAL_DIR_SIZE_DESC = "size of old WALs directory in bytes"; /** * Increment the number of requests the cluster has seen. diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java index 051ad4335c28..ab8b4f5d9491 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java @@ -146,4 +146,9 @@ public interface MetricsMasterWrapper { * Get the time in Millis when the master finished initializing/becoming the active master */ long getMasterInitializationTime(); + + /** + * Get the size of old WALs directory in bytes. + */ + long getOldWALsDirSize(); } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java index 2dbdeff0dd49..f34b79fb3048 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java @@ -106,7 +106,9 @@ public void getMetrics(MetricsCollector metricsCollector, boolean all) { .tag(Interns.info(SERVER_NAME_NAME, SERVER_NAME_DESC), masterWrapper.getServerName()) .tag(Interns.info(CLUSTER_ID_NAME, CLUSTER_ID_DESC), masterWrapper.getClusterId()) .tag(Interns.info(IS_ACTIVE_MASTER_NAME, IS_ACTIVE_MASTER_DESC), - String.valueOf(masterWrapper.getIsActiveMaster())); + String.valueOf(masterWrapper.getIsActiveMaster())) + .addGauge(Interns.info(OLD_WAL_DIR_SIZE_NAME, OLD_WAL_DIR_SIZE_DESC), + masterWrapper.getOldWALsDirSize()); } metricsRegistry.snapshot(metricsRecordBuilder, all); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 92c65da16000..15038da705d8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -421,6 +421,7 @@ public class HMaster extends HRegionServer implements MasterServices { private SpaceQuotaSnapshotNotifier spaceQuotaSnapshotNotifier; private QuotaObserverChore quotaObserverChore; private SnapshotQuotaObserverChore snapshotQuotaChore; + private OldWALsDirSizeChore oldWALsDirSizeChore; private ProcedureExecutor procedureExecutor; private ProcedureStore procedureStore; @@ -1300,6 +1301,10 @@ private void finishActiveMasterInitialization() throws IOException, InterruptedE this.rollingUpgradeChore = new RollingUpgradeChore(this); getChoreService().scheduleChore(rollingUpgradeChore); + + this.oldWALsDirSizeChore = new OldWALsDirSizeChore(this); + getChoreService().scheduleChore(this.oldWALsDirSizeChore); + status.markComplete("Progress after master initialized complete"); } @@ -1825,6 +1830,7 @@ private void stopChores() { shutdownChore(hbckChore); shutdownChore(regionsRecoveryChore); shutdownChore(rollingUpgradeChore); + shutdownChore(oldWALsDirSizeChore); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java index 125657db1874..d118ed532b43 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java @@ -89,6 +89,9 @@ public boolean accept(Path p) { // create the split log lock private final Lock splitLogLock = new ReentrantLock(); + // old WALs directory size in bytes + private long oldWALsDirSize; + /** * Superceded by {@link SplitWALManager}; i.e. procedure-based WAL splitting rather than 'classic' * zk-coordinated WAL splitting. @@ -113,6 +116,7 @@ public MasterWalManager(Configuration conf, FileSystem fs, MasterServices servic this.services = services; this.splitLogManager = new SplitLogManager(services, conf); this.oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME); + this.oldWALsDirSize = 0; } public void stop() { @@ -133,6 +137,14 @@ Path getOldLogDir() { return this.oldLogDir; } + public void updateOldWALsDirSize() throws IOException { + this.oldWALsDirSize = fs.getContentSummary(this.oldLogDir).getLength(); + } + + public long getOldWALsDirSize() { + return this.oldWALsDirSize; + } + public FileSystem getFileSystem() { return this.fs; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java index 97fd79050858..e60f3111a93e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java @@ -209,4 +209,12 @@ public Map> getNamespaceSpaceUtilization() { Entry convertSnapshot(SpaceQuotaSnapshot snapshot) { return new SimpleImmutableEntry(snapshot.getUsage(), snapshot.getLimit()); } + + @Override + public long getOldWALsDirSize() { + if (master == null || !master.isInitialized()) { + return 0; + } + return master.getMasterWalManager().getOldWALsDirSize(); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/OldWALsDirSizeChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/OldWALsDirSizeChore.java new file mode 100644 index 000000000000..b2f0622b7d28 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/OldWALsDirSizeChore.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import java.io.IOException; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.ScheduledChore; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This chore is used to update the 'oldWALsDirSize' variable in {@link MasterWalManager} through + * the {@link MasterWalManager#updateOldWALsDirSize()} method. + */ +@InterfaceAudience.Private +public class OldWALsDirSizeChore extends ScheduledChore { + private static final Logger LOG = LoggerFactory.getLogger(OldWALsDirSizeChore.class); + + private final MasterServices master; + + public OldWALsDirSizeChore(MasterServices master) { + super(master.getServerName() + "-OldWALsDirSizeChore", master, + master.getConfiguration().getInt(HConstants.HBASE_OLDWAL_DIR_SIZE_UPDATER_PERIOD, + HConstants.DEFAULT_HBASE_OLDWAL_DIR_SIZE_UPDATER_PERIOD)); + this.master = master; + } + + @Override + protected void chore() { + try { + this.master.getMasterWalManager().updateOldWALsDirSize(); + } catch (IOException e) { + LOG.error("Got exception while trying to update the old WALs Directory size counter: " + + e.getMessage(), e); + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java index 7500c8749352..078dd5d4a55f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java @@ -186,6 +186,8 @@ public void testDefaultMasterMetrics() throws Exception { metricsHelper.assertCounter(MetricsMasterSource.SERVER_CRASH_METRIC_PREFIX + "SubmittedCount", 0, masterSource); + metricsHelper.assertGauge("oldWALsDirSize", master.getMasterWalManager().getOldWALsDirSize(), + masterSource); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java index c37cbc773452..104eee4f1f66 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java @@ -74,6 +74,7 @@ public void testInfo() throws IOException { assertEquals(master.getMasterCoprocessors().length, info.getCoprocessors().length); assertEquals(master.getServerManager().getOnlineServersList().size(), info.getNumRegionServers()); + assertEquals(master.getMasterWalManager().getOldWALsDirSize(), info.getOldWALsDirSize()); int regionServerCount = NUM_RS + (LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration()) ? 1 : 0); assertEquals(regionServerCount, info.getNumRegionServers()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestOldWALsDirSizeChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestOldWALsDirSizeChore.java new file mode 100644 index 000000000000..469527a75a37 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestOldWALsDirSizeChore.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.master.assignment.MockMasterServices; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tests for OldWALsDirSizeChore Here we are using the {@link MockMasterServices} to mock the Hbase + * Master. Chore's won't be running automatically; we need to run every time. + */ +@Category({ MasterTests.class, SmallTests.class }) +public class TestOldWALsDirSizeChore { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestOldWALsDirSizeChore.class); + + private static final Logger LOG = LoggerFactory.getLogger(TestOldWALsDirSizeChore.class); + + private MockMasterServices master; + + private static final HBaseTestingUtility HTU = new HBaseTestingUtility(); + + @Before + public void setUp() throws Exception { + master = new MockMasterServices(HTU.getConfiguration()); + master.start(10, null); + } + + @After + public void tearDown() throws Exception { + master.stop("tearDown"); + } + + @Test + public void testOldWALsDirSizeChore() throws IOException { + // Assume the OldWALs directory size is initially zero as the chore hasn't run yet + long currentOldWALsDirSize = master.getMasterWalManager().getOldWALsDirSize(); + assertEquals("Initial OldWALs directory size should be zero before running the chore", 0, + currentOldWALsDirSize); + + int dummyFileSize = 50 * 1024 * 1024; // 50MB + byte[] dummyData = new byte[dummyFileSize]; + + // Create a dummy file in the OldWALs directory + Path dummyFileInOldWALsDir = new Path(master.getMasterWalManager().getOldLogDir(), "dummy.txt"); + try (FSDataOutputStream outputStream = + master.getMasterWalManager().getFileSystem().create(dummyFileInOldWALsDir)) { + outputStream.write(dummyData); + } + + // Run the OldWALsDirSizeChore to update the directory size + OldWALsDirSizeChore oldWALsDirSizeChore = new OldWALsDirSizeChore(master); + oldWALsDirSizeChore.chore(); + + // Verify that the OldWALs directory size has increased by the file size + assertEquals("OldWALs directory size after chore should be as expected", dummyFileSize, + master.getMasterWalManager().getOldWALsDirSize()); + } +}