From 3e8a53498d126e6ff7a89e26eef7d949672e8b89 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 9 May 2024 16:03:09 +0800 Subject: [PATCH] [BugFix] Fix dynamic partition table unexpectly stop scheduling (backport #45235) (#45313) Signed-off-by: yiming Signed-off-by: Dejun Xia Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: yiming (cherry picked from commit 5ef58ac7d6088572ea2f9ffbc334e80d634ec426) # Conflicts: # fe/fe-core/src/main/java/com/starrocks/clone/DynamicPartitionScheduler.java # fe/fe-core/src/main/java/com/starrocks/common/util/DynamicPartitionUtil.java # fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java --- .../java/com/starrocks/catalog/Database.java | 5 + .../catalog/DynamicPartitionProperty.java | 22 +- .../java/com/starrocks/catalog/OlapTable.java | 2 +- .../clone/DynamicPartitionScheduler.java | 227 +++++++++++++++++- .../CurrentQueryBackendInstanceProcDir.java | 2 +- .../common/util/DynamicPartitionUtil.java | 80 ++++-- .../java/com/starrocks/qe/ShowExecutor.java | 5 +- .../com/starrocks/server/LocalMetastore.java | 14 ++ .../sql/ast/ShowDynamicPartitionStmt.java | 1 + .../java/com/starrocks/alter/AlterTest.java | 4 +- .../alter/SchemaChangeJobV2Test.java | 6 +- .../starrocks/catalog/TablePropertyTest.java | 2 +- 12 files changed, 324 insertions(+), 46 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/Database.java b/fe/fe-core/src/main/java/com/starrocks/catalog/Database.java index 7e0f1e0e08304..13c3c68dd166f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/Database.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/Database.java @@ -44,6 +44,7 @@ import com.starrocks.persist.CreateTableInfo; import com.starrocks.persist.DropInfo; import com.starrocks.server.GlobalStateMgr; +import com.starrocks.statistic.StatsConstants; import com.starrocks.system.SystemInfoService; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -863,6 +864,10 @@ public boolean isInfoSchemaDb() { return fullQualifiedName.equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME); } + public boolean isStatisticsDatabase() { + return fullQualifiedName.equalsIgnoreCase(StatsConstants.STATISTICS_DB_NAME); + } + // the invoker should hold db's writeLock public void setExist(boolean exist) { this.exist = exist; diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/DynamicPartitionProperty.java b/fe/fe-core/src/main/java/com/starrocks/catalog/DynamicPartitionProperty.java index 6ffb3ac469fce..3bcd833ac8676 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/DynamicPartitionProperty.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/DynamicPartitionProperty.java @@ -49,9 +49,9 @@ public class DynamicPartitionProperty { public static final int NOT_SET_HISTORY_PARTITION_NUM = 0; public static final String NOT_SET_PREFIX = "p"; - private boolean exist; + private final boolean exists; - private boolean enable; + private boolean enabled; private String timeUnit; private int start; private int end; @@ -64,8 +64,8 @@ public class DynamicPartitionProperty { private int historyPartitionNum; public DynamicPartitionProperty(Map properties) { if (properties != null && !properties.isEmpty()) { - this.exist = true; - this.enable = Boolean.parseBoolean(properties.get(ENABLE)); + this.exists = true; + this.enabled = Boolean.parseBoolean(properties.get(ENABLE)); this.timeUnit = properties.get(TIME_UNIT); this.tz = TimeUtils.getOrSystemTimeZone(properties.get(TIME_ZONE)); // In order to compatible dynamic add partition version @@ -79,7 +79,7 @@ public DynamicPartitionProperty(Map properties) { HISTORY_PARTITION_NUM, String.valueOf(NOT_SET_HISTORY_PARTITION_NUM))); createStartOfs(properties); } else { - this.exist = false; + this.exists = false; } } @@ -99,8 +99,8 @@ private void createStartOfs(Map properties) { } } - public boolean isExist() { - return exist; + public boolean isExists() { + return exists; } public String getTimeUnit() { @@ -123,8 +123,8 @@ public int getBuckets() { return buckets; } - public boolean getEnable() { - return enable; + public boolean isEnabled() { + return enabled; } public StartOfDate getStartOfWeek() { @@ -161,7 +161,7 @@ public int getHistoryPartitionNum() { public String getPropString() { StringBuilder sb = new StringBuilder(); sb.append("{"); - sb.append(ENABLE + ":" + enable + ","); + sb.append(ENABLE + ":" + enabled + ","); sb.append(TIME_UNIT + ":" + timeUnit + ","); sb.append(TIME_ZONE + ":" + tz.getID() + ","); sb.append(START + ":" + start + ","); @@ -190,7 +190,7 @@ public void setTimeUnit(String timeUnit) { @Override public String toString() { - String res = ",\n\"" + ENABLE + "\" = \"" + enable + "\"" + String res = ",\n\"" + ENABLE + "\" = \"" + enabled + "\"" + ",\n\"" + TIME_UNIT + "\" = \"" + timeUnit + "\"" + ",\n\"" + TIME_ZONE + "\" = \"" + tz.getID() + "\"" + ",\n\"" + START + "\" = \"" + start + "\"" diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java index 2ea5869792017..bfc1745c51f9a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java @@ -311,7 +311,7 @@ public TableProperty getTableProperty() { public boolean dynamicPartitionExists() { return tableProperty != null && tableProperty.getDynamicPartitionProperty() != null - && tableProperty.getDynamicPartitionProperty().isExist(); + && tableProperty.getDynamicPartitionProperty().isExists(); } public void setBaseIndexId(long baseIndexId) { diff --git a/fe/fe-core/src/main/java/com/starrocks/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/com/starrocks/clone/DynamicPartitionScheduler.java index 47c2d7c3a0df0..95d04839c9412 100644 --- a/fe/fe-core/src/main/java/com/starrocks/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/com/starrocks/clone/DynamicPartitionScheduler.java @@ -98,15 +98,18 @@ public class DynamicPartitionScheduler extends LeaderDaemon { // (DbId, TableId) for a collection of objects marked with partition_ttl_number > 0 on the table private final Set> ttlPartitionInfo = Sets.newConcurrentHashSet(); - private boolean initialize; + private long lastFindingTime = -1; public enum State { NORMAL, ERROR } + public boolean isInScheduler(long dbId, long tableId) { + return dynamicPartitionTableInfo.contains(new Pair<>(dbId, tableId)); + } + public DynamicPartitionScheduler(String name, long intervalMs) { super(name, intervalMs); - this.initialize = false; } public void registerDynamicPartitionTable(Long dbId, Long tableId) { @@ -318,7 +321,7 @@ private ArrayList getDropPartitionClause(Database db, OlapT return dropPartitionClauses; } - private void executeDynamicPartition() { + private void scheduleDynamicPartition() { Iterator> iterator = dynamicPartitionTableInfo.iterator(); OUTER: while (iterator.hasNext()) { Pair tableInfo = iterator.next(); @@ -608,7 +611,7 @@ public boolean executeDynamicPartitionForTable(Long dbId, Long tableId) { olapTable = (OlapTable) db.getTable(tableId); // Only OlapTable has DynamicPartitionProperty if (olapTable == null || !olapTable.dynamicPartitionExists() || - !olapTable.getTableProperty().getDynamicPartitionProperty().getEnable()) { + !olapTable.getTableProperty().getDynamicPartitionProperty().isEnabled()) { if (olapTable == null) { LOG.warn("Automatically removes the schedule because table does not exist, " + "tableId: {}", tableId); @@ -687,6 +690,201 @@ public boolean executeDynamicPartitionForTable(Long dbId, Long tableId) { return false; } +<<<<<<< HEAD +======= + private void scheduleTTLPartition() { + Iterator> iterator = ttlPartitionInfo.iterator(); + while (iterator.hasNext()) { + Pair tableInfo = iterator.next(); + Long dbId = tableInfo.first; + Long tableId = tableInfo.second; + Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + if (db == null) { + iterator.remove(); + LOG.warn("Could not get database={} info. remove it from scheduler", dbId); + continue; + } + Table table = db.getTable(tableId); + OlapTable olapTable; + if (table instanceof OlapTable) { + olapTable = (OlapTable) table; + } else { + iterator.remove(); + LOG.warn("database={}-{}, table={}. is not olap table. remove it from scheduler", + db.getFullName(), dbId, tableId); + continue; + } + + PartitionInfo partitionInfo = olapTable.getPartitionInfo(); + RangePartitionInfo rangePartitionInfo; + if (partitionInfo instanceof RangePartitionInfo) { + rangePartitionInfo = (RangePartitionInfo) olapTable.getPartitionInfo(); + } else { + LOG.warn("currently only support range partition." + + "remove database={}, table={} from scheduler", dbId, tableId); + continue; + } + + if (rangePartitionInfo.getPartitionColumns().size() != 1) { + iterator.remove(); + LOG.warn("currently only support partition with single column. " + + "remove database={}, table={} from scheduler", dbId, tableId); + continue; + } + + int ttlNumber = olapTable.getTableProperty().getPartitionTTLNumber(); + if (Objects.equals(ttlNumber, INVALID)) { + iterator.remove(); + LOG.warn("database={}, table={} have no ttl. remove it from scheduler", dbId, tableId); + continue; + } + + ArrayList dropPartitionClauses = null; + try { + dropPartitionClauses = getDropPartitionClauseByTTL(olapTable, ttlNumber); + } catch (AnalysisException e) { + LOG.warn("database={}-{}, table={}-{} failed to build drop partition statement.", + db.getFullName(), dbId, table.getName(), tableId, e); + } + if (dropPartitionClauses == null) { + continue; + } + + String tableName = olapTable.getName(); + for (DropPartitionClause dropPartitionClause : dropPartitionClauses) { + db.writeLock(); + try { + GlobalStateMgr.getCurrentState().dropPartition(db, olapTable, dropPartitionClause); + clearDropPartitionFailedMsg(tableName); + } catch (DdlException e) { + recordDropPartitionFailedMsg(db.getOriginName(), tableName, e.getMessage()); + } finally { + db.writeUnlock(); + } + } + + } + } + + + private ArrayList getDropPartitionClauseByTTL(OlapTable olapTable, int ttlNumber) + throws AnalysisException { + ArrayList dropPartitionClauses = new ArrayList<>(); + RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) (olapTable.getPartitionInfo()); + List partitionColumns = rangePartitionInfo.getPartitionColumns(); + + // Currently, materialized views and automatically created partition tables + // only support single-column partitioning. + Preconditions.checkArgument(partitionColumns.size() == 1); + Type partitionType = partitionColumns.get(0).getType(); + List>> candidatePartitionList = Lists.newArrayList(); + + if (partitionType.isDateType()) { + LocalDateTime currentDateTime = LocalDateTime.now(); + PartitionValue currentPartitionValue = new PartitionValue(currentDateTime.format(DateUtils.DATE_FORMATTER_UNIX)); + PartitionKey currentPartitionKey = PartitionKey.createPartitionKey( + ImmutableList.of(currentPartitionValue), partitionColumns); + // For expr partitioning table, always has a shadow partition, we should avoid deleting it. + PartitionKey shadowPartitionKey = PartitionKey.createShadowPartitionKey(partitionColumns); + + Map> idToRange = rangePartitionInfo.getIdToRange(false); + for (Map.Entry> partitionRange : idToRange.entrySet()) { + PartitionKey lowerPartitionKey = partitionRange.getValue().lowerEndpoint(); + + if (lowerPartitionKey.compareTo(shadowPartitionKey) == 0) { + continue; + } + + if (lowerPartitionKey.compareTo(currentPartitionKey) <= 0) { + candidatePartitionList.add(partitionRange); + } + } + } else if (partitionType.isNumericType()) { + candidatePartitionList = new ArrayList<>(rangePartitionInfo.getIdToRange(false).entrySet()); + } else { + throw new AnalysisException("Partition ttl does not support type:" + partitionType); + } + + candidatePartitionList.sort(Comparator.comparing(o -> o.getValue().upperEndpoint())); + + int allPartitionNumber = candidatePartitionList.size(); + if (allPartitionNumber > ttlNumber) { + int dropSize = allPartitionNumber - ttlNumber; + for (int i = 0; i < dropSize; i++) { + Long checkDropPartitionId = candidatePartitionList.get(i).getKey(); + Partition partition = olapTable.getPartition(checkDropPartitionId); + if (partition != null) { + String dropPartitionName = partition.getName(); + dropPartitionClauses.add(new DropPartitionClause(false, dropPartitionName, + false, true)); + } + } + } + return dropPartitionClauses; + } + + private void recordCreatePartitionFailedMsg(String dbName, String tableName, String msg) { + LOG.warn("dynamic add partition failed: {}, db: {}, table: {}", msg, dbName, tableName); + createOrUpdateRuntimeInfo(tableName, DYNAMIC_PARTITION_STATE, State.ERROR.toString()); + createOrUpdateRuntimeInfo(tableName, CREATE_PARTITION_MSG, msg); + } + + private void clearCreatePartitionFailedMsg(String tableName) { + createOrUpdateRuntimeInfo(tableName, DYNAMIC_PARTITION_STATE, State.NORMAL.toString()); + createOrUpdateRuntimeInfo(tableName, CREATE_PARTITION_MSG, DEFAULT_RUNTIME_VALUE); + } + + private void recordDropPartitionFailedMsg(String dbName, String tableName, String msg) { + LOG.warn("dynamic drop partition failed: {}, db: {}, table: {}", msg, dbName, tableName); + createOrUpdateRuntimeInfo(tableName, DYNAMIC_PARTITION_STATE, State.ERROR.toString()); + createOrUpdateRuntimeInfo(tableName, DROP_PARTITION_MSG, msg); + } + + private void clearDropPartitionFailedMsg(String tableName) { + createOrUpdateRuntimeInfo(tableName, DYNAMIC_PARTITION_STATE, State.NORMAL.toString()); + createOrUpdateRuntimeInfo(tableName, DROP_PARTITION_MSG, DEFAULT_RUNTIME_VALUE); + } + + private void findSchedulableTables() { + Map> dynamicPartitionTables = new HashMap<>(); + Map> ttlPartitionTables = new HashMap<>(); + long start = System.currentTimeMillis(); + for (Long dbId : GlobalStateMgr.getCurrentState().getDbIds()) { + Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + if (db == null) { + continue; + } + if (db.isSystemDatabase() || db.isStatisticsDatabase()) { + continue; + } + + db.readLock(); + try { + for (Table table : GlobalStateMgr.getCurrentState().getDb(dbId).getTables()) { + if (DynamicPartitionUtil.isDynamicPartitionTable(table)) { + registerDynamicPartitionTable(db.getId(), table.getId()); + dynamicPartitionTables.computeIfAbsent(db.getFullName(), k -> new ArrayList<>()) + .add(table.getName()); + } else if (DynamicPartitionUtil.isTTLPartitionTable(table)) { + // Table(MV) with dynamic partition enabled should not specify partition_ttl_number(MV) or + // partition_live_number property. + registerTtlPartitionTable(db.getId(), table.getId()); + ttlPartitionTables.computeIfAbsent(db.getFullName(), k -> new ArrayList<>()) + .add(table.getName()); + } + } + } finally { + db.readUnlock(); + } + } + LOG.info("finished to find all schedulable tables, cost: {}ms, " + + "dynamic partition tables: {}, ttl partition tables: {}, scheduler enabled: {}, scheduler interval: {}s", + System.currentTimeMillis() - start, dynamicPartitionTables, ttlPartitionTables, + Config.dynamic_partition_enable, Config.dynamic_partition_check_interval_seconds); + lastFindingTime = System.currentTimeMillis(); + } + +>>>>>>> 5ef58ac7d6 ([BugFix] Fix dynamic partition table unexpectly stop scheduling (backport #45235) (#45313)) @VisibleForTesting public void runOnceForTest() { runAfterCatalogReady(); @@ -694,14 +892,25 @@ public void runOnceForTest() { @Override protected void runAfterCatalogReady() { - if (!initialize) { - // check Dynamic Partition tables only when FE start - initDynamicPartitionTable(); + // Find all tables that need to be scheduled. + long now = System.currentTimeMillis(); + if ((now - lastFindingTime) > Math.max(300000, Config.dynamic_partition_check_interval_seconds)) { + findSchedulableTables(); } + + // Update scheduler interval. setInterval(Config.dynamic_partition_check_interval_seconds * 1000L); + + // Schedule tables with dynamic partition enabled(only works for base table). if (Config.dynamic_partition_enable) { - executeDynamicPartition(); + scheduleDynamicPartition(); } - executePartitionTimeToLive(); + + // Schedule tables(mvs) with ttl partition enabled. + // For now, partition_live_number works for base table with + // single column range partitioning(including expr partitioning, e.g. ... partition by date_trunc('month', col). + // partition_ttl_number and partition_ttl work for mv with + // single column range partitioning(including expr partitioning). + scheduleTTLPartition(); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/common/proc/CurrentQueryBackendInstanceProcDir.java b/fe/fe-core/src/main/java/com/starrocks/common/proc/CurrentQueryBackendInstanceProcDir.java index a1bdfe5369b4c..2e9724aa2da93 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/proc/CurrentQueryBackendInstanceProcDir.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/proc/CurrentQueryBackendInstanceProcDir.java @@ -44,7 +44,7 @@ public boolean register(String name, ProcNodeInterface node) { @Override public ProcNodeInterface lookup(String name) throws AnalysisException { - throw new AnalysisException(name + " does't exist."); + throw new AnalysisException(name + " doesn't exist."); } @Override diff --git a/fe/fe-core/src/main/java/com/starrocks/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/com/starrocks/common/util/DynamicPartitionUtil.java index 5cbe2f211d566..be291262db769 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/util/DynamicPartitionUtil.java @@ -142,7 +142,7 @@ public static void checkStartDayOfMonth(String val) throws DdlException { } try { int dayOfMonth = Integer.parseInt(val); - // only support from 1st to 28th, not allow 29th, 30th and 31th to avoid problems + // only support from 1st to 28th, not allow 29th, 30th and 31st to avoid problems // caused by lunar year and lunar month if (dayOfMonth < 1 || dayOfMonth > 28) { throw new DdlException(DynamicPartitionProperty.START_DAY_OF_MONTH + " should between 1 and 28"); @@ -258,7 +258,7 @@ public static void registerOrRemovePartitionScheduleInfo(long dbId, OlapTable ol public static void registerOrRemoveDynamicPartitionTable(long dbId, OlapTable olapTable) { if (olapTable.getTableProperty() != null && olapTable.getTableProperty().getDynamicPartitionProperty() != null) { - if (olapTable.getTableProperty().getDynamicPartitionProperty().getEnable()) { + if (olapTable.getTableProperty().getDynamicPartitionProperty().isEnabled()) { GlobalStateMgr.getCurrentState().getDynamicPartitionScheduler() .registerDynamicPartitionTable(dbId, olapTable.getId()); } else { @@ -390,24 +390,20 @@ public static Map analyzeDynamicPartition(Map pr public static void checkAlterAllowed(OlapTable olapTable) throws DdlException { TableProperty tableProperty = olapTable.getTableProperty(); if (tableProperty != null && tableProperty.getDynamicPartitionProperty() != null && - tableProperty.getDynamicPartitionProperty().isExist() && - tableProperty.getDynamicPartitionProperty().getEnable()) { + tableProperty.getDynamicPartitionProperty().isExists() && + tableProperty.getDynamicPartitionProperty().isEnabled()) { throw new DdlException("Cannot add/drop partition on a Dynamic Partition Table, " + "Use command `ALTER TABLE tbl_name SET (\"dynamic_partition.enable\" = \"false\")` firstly."); } } - public static boolean isDynamicPartitionTable(Table table) { - if (!(table instanceof OlapTable) || - !(((OlapTable) table).getPartitionInfo().getType().equals(PartitionType.RANGE))) { - return false; - } - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) ((OlapTable) table).getPartitionInfo(); - TableProperty tableProperty = ((OlapTable) table).getTableProperty(); - if (tableProperty == null || !tableProperty.getDynamicPartitionProperty().isExist()) { + + private static boolean isTableSchedulable(Table table, boolean checkingDynamicPartitionTable) { + if (!(table instanceof OlapTable)) { return false; } +<<<<<<< HEAD return rangePartitionInfo.getPartitionColumns().size() == 1 && tableProperty.getDynamicPartitionProperty().getEnable(); } @@ -425,6 +421,58 @@ public static boolean isTTLPartitionTable(Table table) { RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) ((OlapTable) table).getPartitionInfo(); return rangePartitionInfo.getPartitionColumns().size() == 1; +======= + OlapTable olapTable = (OlapTable) table; + TableProperty tableProperty = olapTable.getTableProperty(); + if (tableProperty == null) { + return false; + } + + PartitionInfo partitionInfo = olapTable.getPartitionInfo(); + PartitionType partitionType = partitionInfo.getType(); + int partitionColumnSize = -1; + boolean result = partitionType.equals(PartitionType.RANGE); + if (result) { + RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) olapTable.getPartitionInfo(); + partitionColumnSize = rangePartitionInfo.getPartitionColumns().size(); + if (partitionColumnSize != 1) { + result = false; + } + } + + if (checkingDynamicPartitionTable) { + if (!tableProperty.getDynamicPartitionProperty().isExists() || + !tableProperty.getDynamicPartitionProperty().isEnabled()) { + return false; + } + if (!result) { + LOG.info("olap table {}-{} with dynamic partition enabled, but unable to schedule, " + + "partition type: {}, partition column size: {}", + table.getName(), table.getId(), partitionType, partitionColumnSize); + } + } else { + Map properties = tableProperty.getProperties(); + if (!properties.containsKey(PROPERTIES_PARTITION_TTL_NUMBER) && + !properties.containsKey(PROPERTIES_PARTITION_LIVE_NUMBER)) { + return false; + } + if (!result) { + LOG.info("olap table {}-{} with partition ttl enabled, but unable to schedule, " + + "partition type: {}, partition column size: {}", + table.getName(), table.getId(), partitionType, partitionColumnSize); + } + } + + return true; + } + + public static boolean isDynamicPartitionTable(Table table) { + return isTableSchedulable(table, true); + } + + public static boolean isTTLPartitionTable(Table table) { + return isTableSchedulable(table, false); +>>>>>>> 5ef58ac7d6 ([BugFix] Fix dynamic partition table unexpectly stop scheduling (backport #45235) (#45313)) } @@ -508,7 +556,7 @@ public static String getPartitionRangeString(DynamicPartitionProperty property, /** * return formatted string of partition range in HOUR granularity. - * offset: The offset from the current hour. 0 means current hour, -1 means pre hour, 1 means next hour. + * offset: The offset from the current hour. 0 means current hour, -1 means previous hour, 1 means next hour. * format: the format of the return hour string. *

* Eg: @@ -534,7 +582,7 @@ private static String getPartitionRangeOfDay(ZonedDateTime current, int offset, /** * return formatted string of partition range in WEEK granularity. - * offset: The offset from the current week. 0 means current week, 1 means next week, -1 means last week. + * offset: The offset from the current week. 0 means current week, 1 means next week, -1 mean 'last week'. * startOf: Define the start day of each week. 1 means MONDAY, 7 means SUNDAY. * format: the format of the return date string. *

@@ -554,12 +602,12 @@ private static String getPartitionRangeOfWeek(ZonedDateTime current, int offset, /** * return formatted string of partition range in MONTH granularity. - * offset: The offset from the current month. 0 means current month, 1 means next month, -1 means last month. + * offset: The offset from the current month. 0 means current month, 1 means next month, -1 means 'last month'. * startOf: Define the start date of each month. 1 means start on the 1st of every month. * format: the format of the return date string. *

* Eg: - * Today is 2020-05-24, offset = 1, startOf.month = 3 + * Today is 2020-05-24, offset = 1, `startOf.month = 3` * It will return 2020-06-03 */ private static String getPartitionRangeOfMonth(ZonedDateTime current, int offset, StartOfDate startOf, diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java b/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java index 23395bc1d0c1d..12dd4fc72eb00 100644 --- a/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java @@ -1702,7 +1702,7 @@ private void handleShowDynamicPartition() { olapTable.getDefaultReplicationNum() : FeConstants.default_replication_num; rows.add(Lists.newArrayList( tableName, - String.valueOf(dynamicPartitionProperty.getEnable()), + String.valueOf(dynamicPartitionProperty.isEnabled()), dynamicPartitionProperty.getTimeUnit().toUpperCase(), String.valueOf(dynamicPartitionProperty.getStart()), String.valueOf(dynamicPartitionProperty.getEnd()), @@ -1719,7 +1719,8 @@ private void handleShowDynamicPartition() { dynamicPartitionScheduler .getRuntimeInfo(tableName, DynamicPartitionScheduler.CREATE_PARTITION_MSG), dynamicPartitionScheduler - .getRuntimeInfo(tableName, DynamicPartitionScheduler.DROP_PARTITION_MSG))); + .getRuntimeInfo(tableName, DynamicPartitionScheduler.DROP_PARTITION_MSG), + String.valueOf(dynamicPartitionScheduler.isInScheduler(db.getId(), olapTable.getId())))); } } finally { db.readUnlock(); diff --git a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java index f3b8f69a6a936..b57a6d783f6c5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java @@ -2453,6 +2453,20 @@ private void createOlapOrLakeTable(Database db, CreateTableStmt stmt) throws Ddl colocateTableIndex.removeTable(tableId); } } +<<<<<<< HEAD +======= + if (Config.dynamic_partition_enable && table.getTableProperty().getDynamicPartitionProperty().isEnabled()) { + new Thread(() -> { + try { + GlobalStateMgr.getCurrentState().getDynamicPartitionScheduler() + .executeDynamicPartitionForTable(db.getId(), tableId); + } catch (Exception ex) { + LOG.warn("Some problems were encountered in the process of triggering " + + "the execution of dynamic partitioning", ex); + } + }, "BackgroundDynamicPartitionThread").start(); + } +>>>>>>> 5ef58ac7d6 ([BugFix] Fix dynamic partition table unexpectly stop scheduling (backport #45235) (#45313)) } private void processConstraint( diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/ShowDynamicPartitionStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/ShowDynamicPartitionStmt.java index 4d72dd1040f1e..841e2561a718b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/ShowDynamicPartitionStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/ShowDynamicPartitionStmt.java @@ -39,6 +39,7 @@ public class ShowDynamicPartitionStmt extends ShowStmt { .addColumn(new Column("State", ScalarType.createVarchar(20))) .addColumn(new Column("LastCreatePartitionMsg", ScalarType.createVarchar(20))) .addColumn(new Column("LastDropPartitionMsg", ScalarType.createVarchar(20))) + .addColumn(new Column("InScheduler", ScalarType.createVarchar(20))) .build(); public ShowDynamicPartitionStmt(String db) { diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/AlterTest.java b/fe/fe-core/src/test/java/com/starrocks/alter/AlterTest.java index 1596d3ac645e2..3c3300c6e0f18 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/AlterTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/AlterTest.java @@ -543,7 +543,7 @@ public void testConflictAlterOperations() throws Exception { " );"; alterTableWithNewParser(stmt, false); - Assert.assertTrue(tbl.getTableProperty().getDynamicPartitionProperty().getEnable()); + Assert.assertTrue(tbl.getTableProperty().getDynamicPartitionProperty().isEnabled()); Assert.assertEquals(4, tbl.getIndexIdToSchema().size()); // add partition when dynamic partition is enable @@ -560,7 +560,7 @@ public void testConflictAlterOperations() throws Exception { // disable the dynamic partition stmt = "alter table test.tbl1 set ('dynamic_partition.enable' = 'false')"; alterTableWithNewParser(stmt, false); - Assert.assertFalse(tbl.getTableProperty().getDynamicPartitionProperty().getEnable()); + Assert.assertFalse(tbl.getTableProperty().getDynamicPartitionProperty().isEnabled()); // add partition when dynamic partition is disable stmt = "alter table test.tbl1 add partition p3 values less than('2020-04-01') " + diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/SchemaChangeJobV2Test.java b/fe/fe-core/src/test/java/com/starrocks/alter/SchemaChangeJobV2Test.java index 07c22c7e0e591..e9c005b45ef31 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/SchemaChangeJobV2Test.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/SchemaChangeJobV2Test.java @@ -227,8 +227,8 @@ public void testModifyDynamicPartitionNormal() throws UserException { Database db = CatalogMocker.mockDb(); OlapTable olapTable = (OlapTable) db.getTable(CatalogMocker.TEST_TBL2_ID); schemaChangeHandler.process(alterClauses, db, olapTable); - Assert.assertTrue(olapTable.getTableProperty().getDynamicPartitionProperty().isExist()); - Assert.assertTrue(olapTable.getTableProperty().getDynamicPartitionProperty().getEnable()); + Assert.assertTrue(olapTable.getTableProperty().getDynamicPartitionProperty().isExists()); + Assert.assertTrue(olapTable.getTableProperty().getDynamicPartitionProperty().isEnabled()); Assert.assertEquals("day", olapTable.getTableProperty().getDynamicPartitionProperty().getTimeUnit()); Assert.assertEquals(3, olapTable.getTableProperty().getDynamicPartitionProperty().getEnd()); Assert.assertEquals("p", olapTable.getTableProperty().getDynamicPartitionProperty().getPrefix()); @@ -239,7 +239,7 @@ public void testModifyDynamicPartitionNormal() throws UserException { properties.put(DynamicPartitionProperty.ENABLE, "false"); tmpAlterClauses.add(new ModifyTablePropertiesClause(properties)); schemaChangeHandler.process(tmpAlterClauses, db, olapTable); - Assert.assertFalse(olapTable.getTableProperty().getDynamicPartitionProperty().getEnable()); + Assert.assertFalse(olapTable.getTableProperty().getDynamicPartitionProperty().isEnabled()); // set dynamic_partition.time_unit = week tmpAlterClauses = new ArrayList<>(); properties.put(DynamicPartitionProperty.TIME_UNIT, "week"); diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/TablePropertyTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/TablePropertyTest.java index 54ed46042bbc0..8f88ec574e6ed 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/TablePropertyTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/TablePropertyTest.java @@ -64,7 +64,7 @@ public void testNormal() throws IOException { DynamicPartitionProperty readDynamicPartitionProperty = readTableProperty.getDynamicPartitionProperty(); DynamicPartitionProperty dynamicPartitionProperty = new DynamicPartitionProperty(properties); Assert.assertEquals(readTableProperty.getProperties(), properties); - Assert.assertEquals(readDynamicPartitionProperty.getEnable(), dynamicPartitionProperty.getEnable()); + Assert.assertEquals(readDynamicPartitionProperty.isEnabled(), dynamicPartitionProperty.isEnabled()); Assert.assertEquals(readDynamicPartitionProperty.getBuckets(), dynamicPartitionProperty.getBuckets()); Assert.assertEquals(readDynamicPartitionProperty.getPrefix(), dynamicPartitionProperty.getPrefix()); Assert.assertEquals(readDynamicPartitionProperty.getStart(), dynamicPartitionProperty.getStart());