HBASE-28068 Add hbase.normalizer.merge.merge_request_max_number_of_re…

…gions property … (apache#5403) Authored-by: Rahul Kumar <[email protected]> Signed-off-by: Nick Dimiduk <[email protected]>
rahulLiving · Sep 22, 2023 · 387728b · 387728b
1 parent 84ee88d
commit 387728b
Show file tree

Hide file tree

Showing 3 changed files with 85 additions and 14 deletions.
diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
@@ -649,6 +649,12 @@ possible configurations would overwhelm and obscure the important.
     <description>The minimum size for a region to be considered for a merge, in whole
       MBs.</description>
   </property>
+  <property>
+    <name>hbase.normalizer.merge.merge_request_max_number_of_regions</name>
+    <value>50</value>
+    <description>The maximum number of region count in a merge request for merge
+      normalization.</description>
+  </property>
   <property>
     <name>hbase.table.normalization.enabled</name>
     <value>false</value>

diff --git a/...erver/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/...erver/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -73,7 +73,10 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver
   static final String MERGE_MIN_REGION_AGE_DAYS_KEY = "hbase.normalizer.merge.min_region_age.days";
   static final int DEFAULT_MERGE_MIN_REGION_AGE_DAYS = 3;
   static final String MERGE_MIN_REGION_SIZE_MB_KEY = "hbase.normalizer.merge.min_region_size.mb";
-  static final int DEFAULT_MERGE_MIN_REGION_SIZE_MB = 1;
+  static final int DEFAULT_MERGE_MIN_REGION_SIZE_MB = 0;
+  static final String MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY =
+    "hbase.normalizer.merge.merge_request_max_number_of_regions";
+  static final long DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT = 100;
 
   private MasterServices masterServices;
   private NormalizerConfiguration normalizerConfiguration;
@@ -131,6 +134,16 @@ private static long parseMergeMinRegionSizeMb(final Configuration conf) {
     return settledValue;
   }
 
+  private static long parseMergeRequestMaxNumberOfRegionsCount(final Configuration conf) {
+    final long parsedValue = conf.getLong(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY,
+      DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT);
+    final long settledValue = Math.max(2, parsedValue);
+    if (parsedValue != settledValue) {
+      warnInvalidValue(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, parsedValue, settledValue);
+    }
+    return settledValue;
+  }
+
   private static <T> void warnInvalidValue(final String key, final T parsedValue,
     final T settledValue) {
     LOG.warn("Configured value {}={} is invalid. Setting value to {}.", key, parsedValue,
@@ -179,6 +192,10 @@ public long getMergeMinRegionSizeMb() {
     return normalizerConfiguration.getMergeMinRegionSizeMb();
   }
 
+  public long getMergeRequestMaxNumberOfRegionsCount() {
+    return normalizerConfiguration.getMergeRequestMaxNumberOfRegionsCount();
+  }
+
   @Override
   public void setMasterServices(final MasterServices masterServices) {
     this.masterServices = masterServices;
@@ -367,19 +384,21 @@ private List<NormalizationPlan> computeMergeNormalizationPlans(final NormalizeCo
           break;
         }
         if (
-          rangeMembers.isEmpty() // when there are no range members, seed the range with whatever
-                                 // we have. this way we're prepared in case the next region is
-                                 // 0-size.
-            || (rangeMembers.size() == 1 && sumRangeMembersSizeMb == 0) // when there is only one
-                                                                        // region and the size is 0,
-                                                                        // seed the range with
-                                                                        // whatever we have.
-            || regionSizeMb == 0 // always add an empty region to the current range.
-            || (regionSizeMb + sumRangeMembersSizeMb <= avgRegionSizeMb)
-        ) { // add the current region
-            // to the range when
-            // there's capacity
-            // remaining.
+          // when there are no range members, seed the range with whatever we have. this way we're
+          // prepared in case the next region is 0-size.
+          rangeMembers.isEmpty()
+            // when there is only one region and the size is 0, seed the range with whatever we
+            // have.
+            || (rangeMembers.size() == 1 && sumRangeMembersSizeMb == 0)
+            // add an empty region to the current range only if it doesn't exceed max merge request
+            // region count
+            || (regionSizeMb == 0 && rangeMembers.size() < getMergeRequestMaxNumberOfRegionsCount())
+            // add region if current range region size is less than avg region size of table
+            // and current range doesn't exceed max merge request region count
+            || ((regionSizeMb + sumRangeMembersSizeMb <= avgRegionSizeMb)
+              && (rangeMembers.size() < getMergeRequestMaxNumberOfRegionsCount()))
+        ) {
+          // add the current region to the range when there's capacity remaining.
           rangeMembers.add(new NormalizationTarget(regionInfo, regionSizeMb));
           sumRangeMembersSizeMb += regionSizeMb;
           continue;
@@ -479,6 +498,7 @@ private static final class NormalizerConfiguration {
     private final int minRegionCount;
     private final Period mergeMinRegionAge;
     private final long mergeMinRegionSizeMb;
+    private final long mergeRequestMaxNumberOfRegionsCount;
 
     private NormalizerConfiguration() {
       conf = null;
@@ -487,6 +507,7 @@ private NormalizerConfiguration() {
       minRegionCount = DEFAULT_MIN_REGION_COUNT;
       mergeMinRegionAge = Period.ofDays(DEFAULT_MERGE_MIN_REGION_AGE_DAYS);
       mergeMinRegionSizeMb = DEFAULT_MERGE_MIN_REGION_SIZE_MB;
+      mergeRequestMaxNumberOfRegionsCount = DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT;
     }
 
     private NormalizerConfiguration(final Configuration conf,
@@ -497,6 +518,7 @@ private NormalizerConfiguration(final Configuration conf,
       minRegionCount = parseMinRegionCount(conf);
       mergeMinRegionAge = parseMergeMinRegionAge(conf);
       mergeMinRegionSizeMb = parseMergeMinRegionSizeMb(conf);
+      mergeRequestMaxNumberOfRegionsCount = parseMergeRequestMaxNumberOfRegionsCount(conf);
       logConfigurationUpdated(SPLIT_ENABLED_KEY, currentConfiguration.isSplitEnabled(),
         splitEnabled);
       logConfigurationUpdated(MERGE_ENABLED_KEY, currentConfiguration.isMergeEnabled(),
@@ -507,6 +529,9 @@ private NormalizerConfiguration(final Configuration conf,
         currentConfiguration.getMergeMinRegionAge(), mergeMinRegionAge);
       logConfigurationUpdated(MERGE_MIN_REGION_SIZE_MB_KEY,
         currentConfiguration.getMergeMinRegionSizeMb(), mergeMinRegionSizeMb);
+      logConfigurationUpdated(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY,
+        currentConfiguration.getMergeRequestMaxNumberOfRegionsCount(),
+        mergeRequestMaxNumberOfRegionsCount);
     }
 
     public Configuration getConf() {
@@ -558,6 +583,10 @@ public long getMergeMinRegionSizeMb(NormalizeContext context) {
       }
       return mergeMinRegionSizeMb;
     }
+
+    public long getMergeRequestMaxNumberOfRegionsCount() {
+      return mergeRequestMaxNumberOfRegionsCount;
+    }
   }
 
   /**

diff --git a/...r/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java b/...r/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
@@ -22,6 +22,7 @@
 import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_ENABLED_KEY;
 import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_AGE_DAYS_KEY;
 import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_SIZE_MB_KEY;
+import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY;
 import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MIN_REGION_COUNT_KEY;
 import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.SPLIT_ENABLED_KEY;
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -468,6 +469,41 @@ public void testHonorsMergeMinRegionSizeInTD() {
     assertThat(normalizer.computePlansForTable(tableDescriptor), empty());
   }
 
+  @Test
+  public void testHonorsMergeRequestMaxNumberOfRegionsCount() {
+    conf.setBoolean(SPLIT_ENABLED_KEY, false);
+    conf.setInt(MERGE_MIN_REGION_COUNT_KEY, 1);
+    conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
+    conf.setInt(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, 3);
+    final TableName tableName = name.getTableName();
+    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
+    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 0, 1, 0, 1, 0);
+    setupMocksForNormalizer(regionSizes, regionInfos);
+    assertEquals(3, normalizer.getMergeRequestMaxNumberOfRegionsCount());
+    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
+    assertThat(plans,
+      contains(
+        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0)
+          .addTarget(regionInfos.get(1), 1).addTarget(regionInfos.get(2), 0).build(),
+        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(3), 1)
+          .addTarget(regionInfos.get(4), 0).build()));
+  }
+
+  @Test
+  public void testHonorsMergeRequestMaxNumberOfRegionsCountDefault() {
+    conf.setBoolean(SPLIT_ENABLED_KEY, false);
+    conf.setInt(MERGE_MIN_REGION_COUNT_KEY, 1);
+    conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
+    final TableName tableName = name.getTableName();
+    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 3);
+    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 0, 0, 0);
+    setupMocksForNormalizer(regionSizes, regionInfos);
+    assertEquals(50, normalizer.getMergeRequestMaxNumberOfRegionsCount());
+    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
+    assertThat(plans, contains(new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0)
+      .addTarget(regionInfos.get(1), 0).addTarget(regionInfos.get(2), 0).build()));
+  }
+
   @Test
   public void testMergeEmptyRegions0() {
     conf.setBoolean(SPLIT_ENABLED_KEY, false);