From 92efa8475a4406d63b886872d2b60d88fd7258fd Mon Sep 17 00:00:00 2001 From: yujun Date: Fri, 11 Oct 2024 18:41:20 +0800 Subject: [PATCH] update --- .../main/java/org/apache/doris/common/Config.java | 6 ------ .../org/apache/doris/clone/BeLoadRebalancer.java | 10 +++++++++- .../doris/clone/DynamicPartitionScheduler.java | 4 +--- .../doris/catalog/DynamicPartitionTableTest.java | 14 ++++++++++---- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 979c49a51270d4..861968ffb31392 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2533,12 +2533,6 @@ public class Config extends ConfigBase { }) public static long analyze_record_limit = 20000; - @ConfField(mutable = true, masterOnly = true, description = { - "Auto Buckets中预估的压缩数据的倍率", - "the estimated compress factor of partition size in Auto Buckets" - }) - public static int autobucket_compress_size_factor = 5; - @ConfField(mutable = true, masterOnly = true, description = { "Auto Buckets中最小的buckets数目", "min buckets of auto bucket" diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/BeLoadRebalancer.java b/fe/fe-core/src/main/java/org/apache/doris/clone/BeLoadRebalancer.java index 53e8ecf9119a1b..e1460c269c1fe4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/BeLoadRebalancer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/BeLoadRebalancer.java @@ -113,7 +113,8 @@ protected List selectAlternativeTabletsForCluster( numOfLowPaths += pathSlot.getTotalAvailBalanceSlotNum(); } } - LOG.info("get number of low load paths: {}, with medium: {}", numOfLowPaths, medium); + LOG.info("get number of low load paths: {}, with medium: {}, tag: {}, isUrgent {}", + numOfLowPaths, medium, clusterStat.getTag(), isUrgent); List alternativeTabletInfos = Lists.newArrayList(); int clusterAvailableBEnum = infoService.getAllBackendIds(true).size(); @@ -121,6 +122,8 @@ protected List selectAlternativeTabletsForCluster( .map(beStat -> Sets.newHashSet(invertedIndex.getTabletIdsByBackendId(beStat.getBeId()))) .collect(Collectors.toList()); + boolean hasCandidateTablet = false; + // choose tablets from high load backends. // BackendLoadStatistic is sorted by load score in ascend order, // so we need to traverse it from last to first @@ -222,6 +225,8 @@ protected List selectAlternativeTabletsForCluster( continue; } + hasCandidateTablet = true; + // for urgent disk, pick tablets order by size, // then it may always pick tablets that was on the low backends. if (!lowBETablets.isEmpty() @@ -270,6 +275,9 @@ protected List selectAlternativeTabletsForCluster( if (!alternativeTablets.isEmpty()) { LOG.info("select alternative tablets, medium: {}, is urgent: {}, num: {}, detail: {}", medium, isUrgent, alternativeTablets.size(), alternativeTabletInfos); + } else if (isUrgent && !hasCandidateTablet) { + LOG.info("urgent balance cann't found candidate tablets. medium: {}, tag: {}", + medium, clusterStat.getTag()); } return alternativeTablets; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index 20688c57a46866..37a4b922023bfb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -234,9 +234,7 @@ private static int getBucketsNum(DynamicPartitionProperty property, OlapTable ta .collect(Collectors.toCollection(ArrayList::new)); long estimatePartitionSize = getNextPartitionSize(partitionSizeArray); // plus 5 for uncompressed data - // replica's actual disk usage is a litter bigger then its reported data size - // but 5 times maybe a little too big, i don't known why use so big. just add a config here. - long uncompressedPartitionSize = estimatePartitionSize * Config.autobucket_compress_size_factor; + long uncompressedPartitionSize = estimatePartitionSize * 5; int bucketsNum = AutoBucketUtils.getBucketsNum(uncompressedPartitionSize, Config.autobucket_min_buckets); LOG.info("autobucket calc with {} history partitions, table: [{}-{}], partition: {}, buckets num: {}, " + " estimate partition size: {}, last partitions(partition name, local size, remote size): {}", diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index 538383e69b8bd5..2ae051e4f2518e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -1766,20 +1766,25 @@ public void testAutoBuckets() throws Exception { table.readLock(); try { - // first 40 partitions with size 0, then 10 partitions with size 20GB - for (int i = 0; i < 50; i++) { + // first 40 partitions with size 0, then 13 partitions with size 100GB(10GB * 10 buckets) + for (int i = 0; i < 52; i++) { Partition partition = partitions.get(i); partition.updateVisibleVersion(2L); for (MaterializedIndex idx : partition.getMaterializedIndices( MaterializedIndex.IndexExtState.VISIBLE)) { + Assert.assertEquals(10, idx.getTablets().size()); for (Tablet tablet : idx.getTablets()) { for (Replica replica : tablet.getReplicas()) { replica.updateVersion(2L); - replica.setDataSize(i < 40 ? 0L : 20L << 30); + replica.setDataSize(i < 40 ? 0L : 10L << 30); replica.setRowCount(1000L); } } } + if (i >= 40) { + // first 52 partitions are 10 buckets(FeConstants.default_bucket_num) + Assert.assertEquals(10 * (10L << 30), partition.getAllDataSize(true)); + } } } finally { table.readUnlock(); @@ -1793,6 +1798,7 @@ public void testAutoBuckets() throws Exception { partitions = Lists.newArrayList(table.getAllPartitions()); partitions.sort(Comparator.comparing(Partition::getId)); Assert.assertEquals(54, partitions.size()); - Assert.assertTrue(partitions.get(partitions.size() - 1).getDistributionInfo().getBucketNum() > 40); + // 100GB total, 1GB per bucket, should 100 buckets. + Assert.assertEquals(100, partitions.get(partitions.size() - 1).getDistributionInfo().getBucketNum()); } }