From 4f45fefea28b4591e57eef5ae1d31cecde66b6cc Mon Sep 17 00:00:00 2001 From: zhangdong Date: Fri, 6 Dec 2024 17:17:32 +0800 Subject: [PATCH] [enhance](mtmv)Optimize MTMV lock logic (#44967) (#45076) pick: https://github.com/apache/doris/pull/44967 --- .../java/org/apache/doris/catalog/Env.java | 46 +++++++++++-------- .../java/org/apache/doris/catalog/MTMV.java | 2 +- .../org/apache/doris/catalog/OlapTable.java | 20 ++++---- .../apache/doris/mtmv/MTMVPartitionUtil.java | 16 ++----- .../commands/info/ShowCreateMTMVInfo.java | 13 ++---- .../org/apache/doris/qe/ShowExecutor.java | 12 ++--- 6 files changed, 52 insertions(+), 57 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index a0624c407ae426..cf0885e6ec29ea 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -3217,25 +3217,33 @@ public static void getSyncedDdlStmt(TableIf table, List createTableStmt, hidePassword, false, specificVersion, false, true); } - public static String getMTMVDdl(MTMV mtmv) { - StringBuilder sb = new StringBuilder("CREATE MATERIALIZED VIEW "); - sb.append(mtmv.getName()); - addMTMVCols(mtmv, sb); - sb.append("\n"); - sb.append(mtmv.getRefreshInfo()); - addMTMVKeyInfo(mtmv, sb); - addTableComment(mtmv, sb); - addMTMVPartitionInfo(mtmv, sb); - DistributionInfo distributionInfo = mtmv.getDefaultDistributionInfo(); - sb.append("\n").append(distributionInfo.toSql()); - // properties - sb.append("\nPROPERTIES (\n"); - addOlapTablePropertyInfo(mtmv, sb, false, false, null); - addMTMVPropertyInfo(mtmv, sb); - sb.append("\n)"); - sb.append("\nAS "); - sb.append(mtmv.getQuerySql()); - return sb.toString(); + public static String getMTMVDdl(MTMV mtmv) throws AnalysisException { + if (!mtmv.tryReadLock(1, TimeUnit.MINUTES)) { + throw new AnalysisException( + "get table read lock timeout, database=" + mtmv.getQualifiedDbName() + ",table=" + mtmv.getName()); + } + try { + StringBuilder sb = new StringBuilder("CREATE MATERIALIZED VIEW "); + sb.append(mtmv.getName()); + addMTMVCols(mtmv, sb); + sb.append("\n"); + sb.append(mtmv.getRefreshInfo()); + addMTMVKeyInfo(mtmv, sb); + addTableComment(mtmv, sb); + addMTMVPartitionInfo(mtmv, sb); + DistributionInfo distributionInfo = mtmv.getDefaultDistributionInfo(); + sb.append("\n").append(distributionInfo.toSql()); + // properties + sb.append("\nPROPERTIES (\n"); + addOlapTablePropertyInfo(mtmv, sb, false, false, null); + addMTMVPropertyInfo(mtmv, sb); + sb.append("\n)"); + sb.append("\nAS "); + sb.append(mtmv.getQuerySql()); + return sb.toString(); + } finally { + mtmv.readUnlock(); + } } private static void addMTMVKeyInfo(MTMV mtmv, StringBuilder sb) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index 700a1364fdb25d..c1fcc85125e5ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -385,7 +385,7 @@ public Pair>, Map> calculateDoublyPartit Map baseToMv = Maps.newHashMap(); Map> relatedPartitionDescs = MTMVPartitionUtil .generateRelatedPartitionDescs(mvPartitionInfo, mvProperties); - Map mvPartitionItems = getAndCopyPartitionItems(); + Map mvPartitionItems = getAndCopyPartitionItemsWithoutLock(); for (Entry entry : mvPartitionItems.entrySet()) { Set basePartitionNames = relatedPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(), Sets.newHashSet()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 001f36841a7674..21e43179436d97 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -3029,19 +3029,23 @@ public Map getAndCopyPartitionItems() throws AnalysisExce "get table read lock timeout, database=" + getQualifiedDbName() + ",table=" + getName()); } try { - Map res = Maps.newHashMap(); - for (Entry entry : getPartitionInfo().getIdToItem(false).entrySet()) { - Partition partition = idToPartition.get(entry.getKey()); - if (partition != null) { - res.put(partition.getName(), entry.getValue()); - } - } - return res; + return getAndCopyPartitionItemsWithoutLock(); } finally { readUnlock(); } } + public Map getAndCopyPartitionItemsWithoutLock() throws AnalysisException { + Map res = Maps.newHashMap(); + for (Entry entry : getPartitionInfo().getIdToItem(false).entrySet()) { + Partition partition = idToPartition.get(entry.getKey()); + if (partition != null) { + res.put(partition.getName(), entry.getValue()); + } + } + return res; + } + @Override public List getPartitionColumns(Optional snapshot) { return getPartitionColumns(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java index 836a5f08bff2f3..e2775970ab6ab8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java @@ -172,18 +172,12 @@ public static Map> generateRelatedPartitionDescs(M } public static List getPartitionsIdsByNames(MTMV mtmv, List partitions) throws AnalysisException { - mtmv.readLock(); - try { - List res = Lists.newArrayList(); - for (String partitionName : partitions) { - Partition partition = mtmv.getPartitionOrAnalysisException(partitionName); - res.add(partition.getId()); - } - return res; - } finally { - mtmv.readUnlock(); + List res = Lists.newArrayList(); + for (String partitionName : partitions) { + Partition partition = mtmv.getPartitionOrAnalysisException(partitionName); + res.add(partition.getId()); } - + return res; } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ShowCreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ShowCreateMTMVInfo.java index 588af2365741c5..b4fde11c3e1e64 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ShowCreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ShowCreateMTMVInfo.java @@ -74,18 +74,13 @@ public void analyze(ConnectContext ctx) { * @throws DdlException DdlException * @throws IOException IOException */ - public void run(StmtExecutor executor) throws DdlException, IOException { + public void run(StmtExecutor executor) throws DdlException, IOException, org.apache.doris.common.AnalysisException { List> rows = Lists.newArrayList(); Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(mvName.getDb()); MTMV mtmv = (MTMV) db.getTableOrDdlException(mvName.getTbl()); - mtmv.readLock(); - try { - String mtmvDdl = Env.getMTMVDdl(mtmv); - rows.add(Lists.newArrayList(mtmv.getName(), mtmvDdl)); - executor.handleShowCreateMTMVStmt(rows); - } finally { - mtmv.readUnlock(); - } + String mtmvDdl = Env.getMTMVDdl(mtmv); + rows.add(Lists.newArrayList(mtmv.getName(), mtmvDdl)); + executor.handleShowCreateMTMVStmt(rows); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 88762897fae970..741f20dd88fb5e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -1080,15 +1080,9 @@ private void handleShowCreateMTMV() throws AnalysisException { .getDbOrAnalysisException(showStmt.getDb()); MTMV mtmv = (MTMV) db.getTableOrAnalysisException(showStmt.getTable()); List> rows = Lists.newArrayList(); - - mtmv.readLock(); - try { - String mtmvDdl = Env.getMTMVDdl(mtmv); - rows.add(Lists.newArrayList(mtmv.getName(), mtmvDdl)); - resultSet = new ShowResultSet(showStmt.getMetaData(), rows); - } finally { - mtmv.readUnlock(); - } + String mtmvDdl = Env.getMTMVDdl(mtmv); + rows.add(Lists.newArrayList(mtmv.getName(), mtmvDdl)); + resultSet = new ShowResultSet(showStmt.getMetaData(), rows); } // Describe statement