HBASE-28068 : Add hbase.normalizer.merge.merge_request_max_number_of_regions property …

hbase-common/src/main/resources/hbase-default.xml

@@ -656,6 +656,12 @@ possible configurations would overwhelm and obscure the important.
     <description>The minimum size for a region to be considered for a merge, in whole
       MBs.</description>
   </property>
+  <property>
+    <name>hbase.normalizer.merge.merge_request_max_number_of_regions</name>
+    <value>50</value>
+    <description>The maximum number of region count in a merge request for merge
+      normalization.</description>
+  </property>
   <property>
     <name>hbase.table.normalization.enabled</name>
     <value>false</value>

hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java

@@ -81,6 +81,9 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver
   static final int DEFAULT_MERGE_MIN_REGION_AGE_DAYS = 3;
   static final String MERGE_MIN_REGION_SIZE_MB_KEY = "hbase.normalizer.merge.min_region_size.mb";
   static final int DEFAULT_MERGE_MIN_REGION_SIZE_MB = 0;
+  static final String MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY =
+    "hbase.normalizer.merge.merge_request_max_number_of_regions";
+  static final long DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT = 100;
 
   private MasterServices masterServices;
   private NormalizerConfiguration normalizerConfiguration;
@@ -138,6 +141,16 @@ private static long parseMergeMinRegionSizeMb(final Configuration conf) {
     return settledValue;
   }
 
+  private static long parseMergeRequestMaxNumberOfRegionsCount(final Configuration conf) {
+    final long parsedValue = conf.getLong(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY,
+      DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT);
+    final long settledValue = Math.max(2, parsedValue);
+    if (parsedValue != settledValue) {
+      warnInvalidValue(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, parsedValue, settledValue);
+    }
+    return settledValue;
+  }
+
   private static <T> void warnInvalidValue(final String key, final T parsedValue,
     final T settledValue) {
     LOG.warn("Configured value {}={} is invalid. Setting value to {}.", key, parsedValue,
@@ -186,6 +199,10 @@ public long getMergeMinRegionSizeMb() {
     return normalizerConfiguration.getMergeMinRegionSizeMb();
   }
 
+  public long getMergeRequestMaxNumberOfRegionsCount() {
+    return normalizerConfiguration.getMergeRequestMaxNumberOfRegionsCount();
+  }
+
   @Override
   public void setMasterServices(final MasterServices masterServices) {
     this.masterServices = masterServices;
@@ -382,19 +399,21 @@ private List<NormalizationPlan> computeMergeNormalizationPlans(final NormalizeCo
           break;
         }
         if (
-          rangeMembers.isEmpty() // when there are no range members, seed the range with whatever
-                                 // we have. this way we're prepared in case the next region is
-                                 // 0-size.
-            || (rangeMembers.size() == 1 && sumRangeMembersSizeMb == 0) // when there is only one
-                                                                        // region and the size is 0,
-                                                                        // seed the range with
-                                                                        // whatever we have.
-            || regionSizeMb == 0 // always add an empty region to the current range.
-            || (regionSizeMb + sumRangeMembersSizeMb <= avgRegionSizeMb)
-        ) { // add the current region
-            // to the range when
-            // there's capacity
-            // remaining.
+          // when there are no range members, seed the range with whatever we have. this way we're
+          // prepared in case the next region is 0-size.
+          rangeMembers.isEmpty()
+            // when there is only one region and the size is 0, seed the range with whatever we
+            // have.
+            || (rangeMembers.size() == 1 && sumRangeMembersSizeMb == 0)
+            // add an empty region to the current range only if it doesn't exceed max merge request
+            // region count
+            || (regionSizeMb == 0 && rangeMembers.size() < getMergeRequestMaxNumberOfRegionsCount())
+            // add region if current range region size is less than avg region size of table
+            // and current range doesn't exceed max merge request region count
+            || ((regionSizeMb + sumRangeMembersSizeMb <= avgRegionSizeMb)
+              && (rangeMembers.size() < getMergeRequestMaxNumberOfRegionsCount()))
+        ) {
+          // add the current region to the range when there's capacity remaining.
           rangeMembers.add(new NormalizationTarget(regionInfo, regionSizeMb));
           sumRangeMembersSizeMb += regionSizeMb;
           continue;
@@ -502,6 +521,7 @@ private static final class NormalizerConfiguration {
     private final int mergeMinRegionCount;
     private final Period mergeMinRegionAge;
     private final long mergeMinRegionSizeMb;
+    private final long mergeRequestMaxNumberOfRegionsCount;
     private final long cumulativePlansSizeLimitMb;
 
     private NormalizerConfiguration() {
@@ -511,6 +531,7 @@ private NormalizerConfiguration() {
       mergeMinRegionCount = DEFAULT_MERGE_MIN_REGION_COUNT;
       mergeMinRegionAge = Period.ofDays(DEFAULT_MERGE_MIN_REGION_AGE_DAYS);
       mergeMinRegionSizeMb = DEFAULT_MERGE_MIN_REGION_SIZE_MB;
+      mergeRequestMaxNumberOfRegionsCount = DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT;
       cumulativePlansSizeLimitMb = DEFAULT_CUMULATIVE_SIZE_LIMIT_MB;
     }
 
@@ -522,6 +543,7 @@ private NormalizerConfiguration(final Configuration conf,
       mergeMinRegionCount = parseMergeMinRegionCount(conf);
       mergeMinRegionAge = parseMergeMinRegionAge(conf);
       mergeMinRegionSizeMb = parseMergeMinRegionSizeMb(conf);
+      mergeRequestMaxNumberOfRegionsCount = parseMergeRequestMaxNumberOfRegionsCount(conf);
       cumulativePlansSizeLimitMb =
         conf.getLong(CUMULATIVE_SIZE_LIMIT_MB_KEY, DEFAULT_CUMULATIVE_SIZE_LIMIT_MB);
       logConfigurationUpdated(SPLIT_ENABLED_KEY, currentConfiguration.isSplitEnabled(),
@@ -534,6 +556,9 @@ private NormalizerConfiguration(final Configuration conf,
         currentConfiguration.getMergeMinRegionAge(), mergeMinRegionAge);
       logConfigurationUpdated(MERGE_MIN_REGION_SIZE_MB_KEY,
         currentConfiguration.getMergeMinRegionSizeMb(), mergeMinRegionSizeMb);
+      logConfigurationUpdated(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY,
+        currentConfiguration.getMergeRequestMaxNumberOfRegionsCount(),
+        mergeRequestMaxNumberOfRegionsCount);
     }
 
     public Configuration getConf() {
@@ -597,6 +622,10 @@ public long getMergeMinRegionSizeMb(NormalizeContext context) {
       return mergeMinRegionSizeMb;
     }
 
+    public long getMergeRequestMaxNumberOfRegionsCount() {
+      return mergeRequestMaxNumberOfRegionsCount;
+    }
+
     private long getCumulativePlansSizeLimitMb() {
       return cumulativePlansSizeLimitMb;
     }

hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java

@@ -24,6 +24,7 @@
 import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_AGE_DAYS_KEY;
 import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_COUNT_KEY;
 import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_SIZE_MB_KEY;
+import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY;
 import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MIN_REGION_COUNT_KEY;
 import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.SPLIT_ENABLED_KEY;
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -503,6 +504,41 @@ public void testHonorsMergeMinRegionSizeInTD() {
     assertThat(normalizer.computePlansForTable(tableDescriptor), empty());
   }
 
+  @Test
+  public void testHonorsMergeRequestMaxNumberOfRegionsCount() {
+    conf.setBoolean(SPLIT_ENABLED_KEY, false);
+    conf.setInt(MERGE_MIN_REGION_COUNT_KEY, 1);
+    conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
+    conf.setInt(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, 3);
+    final TableName tableName = name.getTableName();
+    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
+    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 0, 1, 0, 1, 0);
+    setupMocksForNormalizer(regionSizes, regionInfos);
+    assertEquals(3, normalizer.getMergeRequestMaxNumberOfRegionsCount());
+    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
+    assertThat(plans,
+      contains(
+        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0)
+          .addTarget(regionInfos.get(1), 1).addTarget(regionInfos.get(2), 0).build(),
+        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(3), 1)
+          .addTarget(regionInfos.get(4), 0).build()));
+  }
+
+  @Test
+  public void testHonorsMergeRequestMaxNumberOfRegionsCountDefault() {
+    conf.setBoolean(SPLIT_ENABLED_KEY, false);
+    conf.setInt(MERGE_MIN_REGION_COUNT_KEY, 1);
+    conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
+    final TableName tableName = name.getTableName();
+    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 3);
+    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 0, 0, 0);
+    setupMocksForNormalizer(regionSizes, regionInfos);
+    assertEquals(50, normalizer.getMergeRequestMaxNumberOfRegionsCount());
+    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
+    assertThat(plans, contains(new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0)
+      .addTarget(regionInfos.get(1), 0).addTarget(regionInfos.get(2), 0).build()));
+  }
+
   @Test
   public void testMergeEmptyRegions0() {
     conf.setBoolean(SPLIT_ENABLED_KEY, false);