From 6b52ca6e0dbe7be7370770a592afd6e8511ee900 Mon Sep 17 00:00:00 2001 From: Jibing Li Date: Wed, 4 Sep 2024 19:56:33 +0800 Subject: [PATCH] Return -1 to neredis if report olap table row count for new table is not done for all tablets. --- be/src/olap/tablet_manager.cpp | 1 + .../doris/analysis/ShowTableStatsStmt.java | 66 ++++++-------- .../doris/catalog/CloudTabletStatMgr.java | 1 + .../doris/catalog/MaterializedIndex.java | 10 +++ .../org/apache/doris/catalog/OlapTable.java | 7 +- .../org/apache/doris/catalog/Replica.java | 10 +++ .../apache/doris/catalog/TabletStatMgr.java | 29 ++++++- .../doris/nereids/stats/StatsCalculator.java | 2 +- .../org/apache/doris/qe/ShowExecutor.java | 9 +- .../doris/service/FrontendServiceImpl.java | 3 + .../apache/doris/statistics/AnalysisInfo.java | 5 -- .../doris/statistics/AnalysisManager.java | 5 ++ .../doris/statistics/BaseAnalysisTask.java | 2 - .../doris/statistics/OlapAnalysisTask.java | 2 +- .../statistics/StatisticsRepository.java | 1 - .../doris/statistics/TableStatsMeta.java | 28 ++---- .../transaction/DatabaseTransactionMgr.java | 3 + gensrc/thrift/BackendService.thrift | 1 + .../suites/statistics/analyze_stats.groovy | 2 +- .../suites/statistics/test_analyze_mv.groovy | 85 ++++++++++++++++--- 20 files changed, 179 insertions(+), 93 deletions(-) diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index 3bac96370333d3..6525e9185dac35 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -1065,6 +1065,7 @@ void TabletManager::build_all_report_tablets_info(std::map* t_tablet_stat.__set_row_count(tablet_info.row_count); t_tablet_stat.__set_total_version_count(tablet_info.total_version_count); t_tablet_stat.__set_visible_version_count(tablet_info.visible_version_count); + t_tablet_stat.__set_visible_version(tablet_info.version); }; for_each_tablet(handler, filter_all_tablets); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java index 54c8f652e125fa..ccffee3086dbea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java @@ -75,8 +75,9 @@ public class ShowTableStatsStmt extends ShowStmt implements NotFallbackInParser new ImmutableList.Builder() .add("table_name") .add("index_name") - .add("row_count") - .add("update_time") + .add("analyze_row_count") + .add("report_row_count") + .add("report_row_count_for_nereids") .build(); private static final ImmutableList COLUMN_PARTITION_TITLE_NAMES = @@ -200,15 +201,15 @@ public long getTableId() { public ShowResultSet constructResultSet(TableStatsMeta tableStatistic, TableIf table) { if (indexName != null) { - return constructIndexResultSet(tableStatistic); + return constructIndexResultSet(tableStatistic, table); } if (partitionNames == null) { return constructTableResultSet(tableStatistic, table); } if (columnNames == null) { - return constructPartitionResultSet(tableStatistic); + return constructPartitionResultSet(tableStatistic, table); } else { - return constructColumnPartitionResultSet(tableStatistic); + return constructColumnPartitionResultSet(tableStatistic, table); } } @@ -216,31 +217,23 @@ public ShowResultSet constructEmptyResultSet() { return new ShowResultSet(getMetaData(), new ArrayList<>()); } - public ShowResultSet constructResultSet(TableIf table) { - List> result = Lists.newArrayList(); - if (partitionNames != null) { - // For partition, return empty result if table stats not exist. - return new ShowResultSet(getMetaData(), result); - } - List row = Lists.newArrayList(); - row.add(""); - row.add(""); - row.add(String.valueOf(table.getCachedRowCount())); - row.add(""); - row.add(""); - row.add(""); - row.add(""); - row.add(""); - row.add(String.valueOf(table.autoAnalyzeEnabled())); - result.add(row); - return new ShowResultSet(getMetaData(), result); - } - public ShowResultSet constructTableResultSet(TableStatsMeta tableStatistic, TableIf table) { - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); if (tableStatistic == null) { - return new ShowResultSet(getMetaData(), new ArrayList<>()); + List> result = Lists.newArrayList(); + List row = Lists.newArrayList(); + row.add(""); + row.add(""); + row.add(String.valueOf(table.getCachedRowCount())); + row.add(""); + row.add(""); + row.add(""); + row.add(""); + row.add(""); + row.add(String.valueOf(table.autoAnalyzeEnabled())); + result.add(row); + return new ShowResultSet(getMetaData(), result); } + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); List> result = Lists.newArrayList(); List row = Lists.newArrayList(); row.add(String.valueOf(tableStatistic.updatedRows)); @@ -260,9 +253,9 @@ public ShowResultSet constructTableResultSet(TableStatsMeta tableStatistic, Tabl return new ShowResultSet(getMetaData(), result); } - public ShowResultSet constructPartitionResultSet(TableStatsMeta tableStatistic) { + public ShowResultSet constructPartitionResultSet(TableStatsMeta tableStatistic, TableIf table) { List> result = Lists.newArrayList(); - if (!(table instanceof OlapTable)) { + if (!(table instanceof OlapTable) || tableStatistic == null) { return new ShowResultSet(getMetaData(), result); } Collection partitions = partitionNames.isStar() @@ -286,7 +279,7 @@ public ShowResultSet constructPartitionResultSet(TableStatsMeta tableStatistic) return new ShowResultSet(getMetaData(), result); } - public ShowResultSet constructIndexResultSet(TableStatsMeta tableStatistic) { + public ShowResultSet constructIndexResultSet(TableStatsMeta tableStatistic, TableIf table) { List> result = Lists.newArrayList(); if (!(table instanceof OlapTable)) { return new ShowResultSet(getMetaData(), result); @@ -296,23 +289,20 @@ public ShowResultSet constructIndexResultSet(TableStatsMeta tableStatistic) { if (indexId == null) { throw new RuntimeException(String.format("Index %s not exist.", indexName)); } - long rowCount = tableStatistic.getRowCount(olapTable.getIndexIdByName(indexName)); - if (rowCount == -1) { - return new ShowResultSet(getMetaData(), result); - } - long updateTime = tableStatistic.getRowCountUpdateTime(olapTable.getIndexIdByName(indexName)); + long rowCount = tableStatistic == null ? -1 : tableStatistic.getRowCount(olapTable.getIndexIdByName(indexName)); List row = Lists.newArrayList(); row.add(table.getName()); row.add(indexName); row.add(String.valueOf(rowCount)); - row.add(String.valueOf(updateTime)); + row.add(String.valueOf(olapTable.getRowCountForIndex(indexId, false))); + row.add(String.valueOf(olapTable.getRowCountForIndex(indexId, true))); result.add(row); return new ShowResultSet(getMetaData(), result); } - public ShowResultSet constructColumnPartitionResultSet(TableStatsMeta tableStatistic) { + public ShowResultSet constructColumnPartitionResultSet(TableStatsMeta tableStatistic, TableIf table) { List> result = Lists.newArrayList(); - if (!(table instanceof OlapTable)) { + if (!(table instanceof OlapTable) || tableStatistic == null) { return new ShowResultSet(getMetaData(), result); } OlapTable olapTable = (OlapTable) table; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java index ff01f736081a70..0ba2686060e1f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java @@ -199,6 +199,7 @@ protected void runAfterCatalogReady() { tableRowsetCount += tabletRowsetCount; tableSegmentCount += tabletSegmentCount; } // end for tablets + index.setRowCountReported(true); index.setRowCount(indexRowCount); } // end for indices } // end for partitions diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java index d4da21753233c5..6e4f6a2248e719 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java @@ -73,6 +73,8 @@ public enum IndexExtState { @SerializedName(value = "rollupFinishedVersion") private long rollupFinishedVersion; + private boolean rowCountReported = false; + public MaterializedIndex() { this.state = IndexState.NORMAL; this.idToTablets = new HashMap<>(); @@ -206,6 +208,14 @@ public int getTabletOrderIdx(long tabletId) { return -1; } + public void setRowCountReported(boolean reported) { + this.rowCountReported = reported; + } + + public boolean getRowCountReported() { + return this.rowCountReported; + } + @Deprecated public void readFields(DataInput in) throws IOException { super.readFields(in); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 4bc05aff20612e..a60da71b299328 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -1564,13 +1564,16 @@ public Set> getColumnIndexPairs(Set columns) { @Override public long fetchRowCount() { - return getRowCountForIndex(baseIndexId); + return getRowCountForIndex(baseIndexId, false); } - public long getRowCountForIndex(long indexId) { + public long getRowCountForIndex(long indexId, boolean strict) { long rowCount = 0; for (Map.Entry entry : idToPartition.entrySet()) { MaterializedIndex index = entry.getValue().getIndex(indexId); + if (strict && !index.getRowCountReported()) { + return -1; + } rowCount += (index == null || index.getRowCount() == -1) ? 0 : index.getRowCount(); } return rowCount; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java index 0fcbef007437aa..f7985e57d00b51 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java @@ -177,6 +177,8 @@ public static class ReplicaContext { private long userDropTime = -1; + private long lastReportVersion = 0; + public Replica() { } @@ -839,4 +841,12 @@ public boolean isScheduleAvailable() { return Env.getCurrentSystemInfo().checkBackendScheduleAvailable(backendId) && !isUserDrop(); } + + public void setLastReportVersion(long version) { + this.lastReportVersion = version; + } + + public long getLastReportVersion() { + return lastReportVersion; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java index 1206c6b2f82ee7..faf1fcab94f2df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java @@ -128,6 +128,7 @@ protected void runAfterCatalogReady() { long version = partition.getVisibleVersion(); for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { long indexRowCount = 0L; + boolean indexReported = true; for (Tablet tablet : index.getTablets()) { Long tabletDataSize = 0L; @@ -135,9 +136,26 @@ protected void runAfterCatalogReady() { Long tabletRowCount = 0L; + boolean tabletReported = false; for (Replica replica : tablet.getReplicas()) { + LOG.debug("Table {} replica {} current version {}, report version {}", + olapTable.getName(), replica.getId(), + replica.getVersion(), replica.getLastReportVersion()); if (replica.checkVersionCatchUp(version, false) - && replica.getRowCount() > tabletRowCount) { + && replica.getRowCount() >= tabletRowCount) { + // 1. If replica version and reported replica version are all equal to + // PARTITION_INIT_VERSION, set tabletReported to true, which indicates this + // tablet is empty for sure when previous report. + // 2. If last report version is larger than PARTITION_INIT_VERSION, set + // tabletReported to true as well. That is, we only guarantee all replicas of + // the tablet are reported for the init version. + // e.g. When replica version is 2, but last reported version is 1, + // tabletReported would be false. + if (replica.getVersion() == Partition.PARTITION_INIT_VERSION + && replica.getLastReportVersion() == Partition.PARTITION_INIT_VERSION + || replica.getLastReportVersion() > Partition.PARTITION_INIT_VERSION) { + tabletReported = true; + } tabletRowCount = replica.getRowCount(); } @@ -157,8 +175,14 @@ protected void runAfterCatalogReady() { tableRowCount += tabletRowCount; indexRowCount += tabletRowCount; + // Only when all tablets of this index are reported, we set indexReported to true. + indexReported = indexReported && tabletReported; } // end for tablets + index.setRowCountReported(indexReported); index.setRowCount(indexRowCount); + LOG.debug("Table {} index {} all tablets reported[{}], row count {}", + olapTable.getName(), olapTable.getIndexNameById(index.getId()), + indexReported, tableRowCount); } // end for indices } // end for partitions @@ -193,6 +217,9 @@ private void updateTabletStat(Long beId, TTabletStatResult result) { replica.setTotalVersionCount(stat.getTotalVersionCount()); replica.setVisibleVersionCount(stat.isSetVisibleVersionCount() ? stat.getVisibleVersionCount() : stat.getTotalVersionCount()); + // Older version BE doesn't set visible version. Set it to max for compatibility. + replica.setLastReportVersion(stat.isSetVisibleVersion() ? stat.getVisibleVersion() + : Long.MAX_VALUE); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index fec744b86b10f2..9ea5811502293a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -366,7 +366,7 @@ private void checkIfUnknownStatsUsedAsKey(StatisticsBuilder builder) { private Statistics computeOlapScan(OlapScan olapScan) { OlapTable olapTable = olapScan.getTable(); - double tableRowCount = olapTable.getRowCountForIndex(olapScan.getSelectedIndexId()); + double tableRowCount = olapTable.getRowCountForIndex(olapScan.getSelectedIndexId(), true); if (tableRowCount <= 0) { AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager(); TableStatsMeta tableMeta = analysisManager.findTableStatsStatus(olapScan.getTable().getId()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 819c98759917cf..c2170f90d5716e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -2725,14 +2725,7 @@ private void handleShowTableStats() { return; } TableStatsMeta tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(tableIf.getId()); - /* - tableStats == null means it's not analyzed, in this case show the estimated row count. - */ - if (tableStats == null) { - resultSet = showTableStatsStmt.constructResultSet(tableIf); - } else { - resultSet = showTableStatsStmt.constructResultSet(tableStats, tableIf); - } + resultSet = showTableStatsStmt.constructResultSet(tableStats, tableIf); } private void handleShowColumnStats() throws AnalysisException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index cc613cb5d8cf76..77f79bc5768478 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -3319,6 +3319,9 @@ public TStatus invalidateStatsCache(TInvalidateFollowerStatsCacheRequest request partitionNames = new PartitionNames(false, new ArrayList<>(target.partitions)); } if (target.isTruncate) { + if (partitionNames == null || partitionNames.isStar() || partitionNames.getPartitionNames() == null) { + tableStats.clearIndexesRowCount(); + } analysisManager.submitAsyncDropStatsTask(target.catalogId, target.dbId, target.tableId, tableStats, partitionNames); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java index 0e89b5225aba03..6ec413821ea37b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java @@ -198,7 +198,6 @@ public enum ScheduleType { public final boolean enablePartition; public final ConcurrentMap indexesRowCount = new ConcurrentHashMap<>(); - public final ConcurrentMap indexesRowCountUpdateTime = new ConcurrentHashMap<>(); public AnalysisInfo(long jobId, long taskId, List taskIds, long catalogId, long dbId, long tblId, Set> jobColumns, Set partitionNames, String colName, Long indexId, @@ -358,8 +357,4 @@ public TableIf getTable() { public void addIndexRowCount(long indexId, long rowCount) { indexesRowCount.put(indexId, rowCount); } - - public void addIndexUpdateRowCountTime(long indexId, long time) { - indexesRowCountUpdateTime.put(indexId, time); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java index 55b0bdc4efd3c7..634f1074cd5658 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java @@ -682,6 +682,9 @@ public void dropStats(TableIf table, PartitionNames partitionNames) throws DdlEx long catalogId = table.getDatabase().getCatalog().getId(); long dbId = table.getDatabase().getId(); long tableId = table.getId(); + if (partitionNames == null || partitionNames.isStar() || partitionNames.getPartitionNames() == null) { + tableStats.clearIndexesRowCount(); + } submitAsyncDropStatsTask(catalogId, dbId, tableId, tableStats, partitionNames); // Drop stats ddl is master only operation. Set partitions = null; @@ -810,6 +813,8 @@ public void invalidateLocalStats(long catalogId, long dbId, long tableId, Set tabletIds = pair.first; long totalRowCount = info.indexId == -1 ? tbl.getRowCount() - : ((OlapTable) tbl).getRowCountForIndex(info.indexId); + : ((OlapTable) tbl).getRowCountForIndex(info.indexId, false); double scaleFactor = (double) totalRowCount / (double) pair.second; // might happen if row count in fe metadata hasn't been updated yet if (Double.isInfinite(scaleFactor) || Double.isNaN(scaleFactor)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java index 83350d16e0423a..0d9fa3674628c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java @@ -392,7 +392,6 @@ public static void alterColumnStatistics(AlterColumnStatsStmt alterColumnStatsSt if (objects.table instanceof OlapTable) { indexId = indexId == -1 ? ((OlapTable) objects.table).getBaseIndexId() : indexId; mockedJobInfo.addIndexRowCount(indexId, (long) Double.parseDouble(rowCount)); - mockedJobInfo.addIndexUpdateRowCountTime(indexId, timestamp); } Env.getCurrentEnv().getAnalysisManager().updateTableStatsForAlterStats(mockedJobInfo, objects.table); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java index 2d566d71cfcbc1..de0f0eed18dae7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java @@ -101,10 +101,7 @@ public class TableStatsMeta implements Writable, GsonPostProcessable { public ConcurrentMap partitionUpdateRows = new ConcurrentHashMap<>(); @SerializedName("irc") - public ConcurrentMap indexesRowCount = new ConcurrentHashMap<>(); - - @SerializedName("ircut") - public ConcurrentMap indexesRowCountUpdateTime = new ConcurrentHashMap<>(); + private ConcurrentMap indexesRowCount = new ConcurrentHashMap<>(); @VisibleForTesting public TableStatsMeta() { @@ -194,8 +191,7 @@ public void update(AnalysisInfo analyzedJob, TableIf tableIf) { if (tableIf != null) { if (tableIf instanceof OlapTable) { indexesRowCount.putAll(analyzedJob.indexesRowCount); - indexesRowCountUpdateTime.putAll(analyzedJob.indexesRowCountUpdateTime); - clearStaleIndexRowCountAndTime((OlapTable) tableIf); + clearStaleIndexRowCount((OlapTable) tableIf); } rowCount = analyzedJob.rowCount; if (rowCount == 0 && AnalysisMethod.SAMPLE.equals(analyzedJob.analysisMethod)) { @@ -227,20 +223,17 @@ public void gsonPostProcess() throws IOException { if (colToColStatsMeta == null) { colToColStatsMeta = new ConcurrentHashMap<>(); } - if (indexesRowCountUpdateTime == null) { - indexesRowCountUpdateTime = new ConcurrentHashMap<>(); - } } public long getRowCount(long indexId) { return indexesRowCount.getOrDefault(indexId, -1L); } - public long getRowCountUpdateTime(long indexId) { - return indexesRowCountUpdateTime.getOrDefault(indexId, 0L); + public void clearIndexesRowCount() { + indexesRowCount.clear(); } - private void clearStaleIndexRowCountAndTime(OlapTable table) { + private void clearStaleIndexRowCount(OlapTable table) { Iterator iterator = indexesRowCount.keySet().iterator(); List indexIds = table.getIndexIds(); while (iterator.hasNext()) { @@ -249,18 +242,11 @@ private void clearStaleIndexRowCountAndTime(OlapTable table) { iterator.remove(); } } - iterator = indexesRowCountUpdateTime.keySet().iterator(); - while (iterator.hasNext()) { - long key = iterator.next(); - if (indexIds.contains(key)) { - iterator.remove(); - } - } } public long getBaseIndexDeltaRowCount(OlapTable table) { - if (colToColStatsMeta == null) { - return -1; + if (colToColStatsMeta == null || colToColStatsMeta.isEmpty() || userInjected) { + return 0; } long maxUpdateRows = 0; String baseIndexName = table.getIndexNameById(table.getBaseIndexId()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java index 5e9c22bede7e82..73c930fb4c6367 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java @@ -2294,6 +2294,9 @@ private boolean updateCatalogAfterVisible(TransactionState transactionState, Dat replica.getId(), newVersion, lastFailedVersion, lastSuccessVersion); } replica.updateVersionWithFailed(newVersion, lastFailedVersion, lastSuccessVersion); + if (newVersion == Partition.PARTITION_INIT_VERSION + 1) { + index.setRowCountReported(false); + } Set partitionIds = backendPartitions.get(replica.getBackendId()); if (partitionIds == null) { partitionIds = Sets.newHashSet(); diff --git a/gensrc/thrift/BackendService.thrift b/gensrc/thrift/BackendService.thrift index 058f84aadc9171..ed0ae243a1ded0 100644 --- a/gensrc/thrift/BackendService.thrift +++ b/gensrc/thrift/BackendService.thrift @@ -38,6 +38,7 @@ struct TTabletStat { 4: optional i64 total_version_count 5: optional i64 remote_data_size 6: optional i64 visible_version_count + 7: optional i64 visible_version } struct TTabletStatResult { diff --git a/regression-test/suites/statistics/analyze_stats.groovy b/regression-test/suites/statistics/analyze_stats.groovy index 37f59406d25b87..fcdb66fcbab911 100644 --- a/regression-test/suites/statistics/analyze_stats.groovy +++ b/regression-test/suites/statistics/analyze_stats.groovy @@ -2743,7 +2743,7 @@ PARTITION `p599` VALUES IN (599) assertEquals("true", alter_result[0][7]) sql """drop stats alter_test""" alter_result = sql """show table stats alter_test""" - assertEquals("false", alter_result[0][7]) + assertEquals("", alter_result[0][7]) sql """alter table alter_test modify column id set stats ('row_count'='100', 'ndv'='0', 'num_nulls'='0.0', 'data_size'='2.69975443E8', 'min_value'='1', 'max_value'='2');""" alter_result = sql """show column stats alter_test(id)""" logger.info("show column alter_test(id) stats: " + alter_result) diff --git a/regression-test/suites/statistics/test_analyze_mv.groovy b/regression-test/suites/statistics/test_analyze_mv.groovy index 4b9644c4821d96..d6ee9b4cfc7c14 100644 --- a/regression-test/suites/statistics/test_analyze_mv.groovy +++ b/regression-test/suites/statistics/test_analyze_mv.groovy @@ -127,39 +127,45 @@ suite("test_analyze_mv") { "replication_num" = "1" ) """ + def result_row + if (!isCloudMode()) { + // Test row count report and report for nereids + result_row = sql """show index stats mvTestDup mvTestDup""" + assertEquals(1, result_row.size()) + assertEquals("mvTestDup", result_row[0][0]) + assertEquals("mvTestDup", result_row[0][1]) + assertEquals("0", result_row[0][3]) + assertEquals("-1", result_row[0][4]) + } + createMV("create materialized view mv1 as select key1 from mvTestDup;") createMV("create materialized view mv2 as select key2 from mvTestDup;") createMV("create materialized view mv3 as select key1, key2, sum(value1), max(value2), min(value3) from mvTestDup group by key1, key2;") sql """insert into mvTestDup values (1, 2, 3, 4, 5), (1, 2, 3, 4, 5), (10, 20, 30, 40, 50), (10, 20, 30, 40, 50), (100, 200, 300, 400, 500), (1001, 2001, 3001, 4001, 5001);""" - def timestamp = System.currentTimeMillis(); sql """analyze table mvTestDup with sync;""" // Test show index row count - def result_row = sql """show index stats mvTestDup mvTestDup""" + result_row = sql """show index stats mvTestDup mvTestDup""" assertEquals(1, result_row.size()) assertEquals("mvTestDup", result_row[0][0]) assertEquals("mvTestDup", result_row[0][1]) assertEquals("6", result_row[0][2]) - assertTrue(Long.parseLong(result_row[0][3]) >= timestamp) result_row = sql """show index stats mvTestDup mv1""" assertEquals(1, result_row.size()) assertEquals("mvTestDup", result_row[0][0]) assertEquals("mv1", result_row[0][1]) assertEquals("6", result_row[0][2]) - assertTrue(Long.parseLong(result_row[0][3]) >= timestamp) result_row = sql """show index stats mvTestDup mv2""" assertEquals(1, result_row.size()) assertEquals("mvTestDup", result_row[0][0]) assertEquals("mv2", result_row[0][1]) assertEquals("6", result_row[0][2]) - assertTrue(Long.parseLong(result_row[0][3]) >= timestamp) result_row = sql """show index stats mvTestDup mv3""" assertEquals(1, result_row.size()) assertEquals("mvTestDup", result_row[0][0]) assertEquals("mv3", result_row[0][1]) assertEquals("4", result_row[0][2]) - assertTrue(Long.parseLong(result_row[0][3]) >= timestamp) // Compare show whole table column stats result with show single column. def result_all = sql """show column stats mvTestDup""" @@ -438,27 +444,21 @@ suite("test_analyze_mv") { assertEquals("FULL", result_sample[0][9]) // Test alter table index row count. - timestamp = System.currentTimeMillis(); sql """alter table mvTestDup modify column `value2` set stats ('row_count'='1.5E8', 'ndv'='3.0', 'num_nulls'='0.0', 'data_size'='1.5E8', 'min_value'='1', 'max_value'='10');""" result_row = sql """show index stats mvTestDup mvTestDup;""" assertEquals("mvTestDup", result_row[0][0]) assertEquals("mvTestDup", result_row[0][1]) assertEquals("150000000", result_row[0][2]) - assertTrue(Long.parseLong(result_row[0][3]) >= timestamp) - timestamp = System.currentTimeMillis(); sql """alter table mvTestDup index mv1 modify column `mv_key1` set stats ('row_count'='3443', 'ndv'='3.0', 'num_nulls'='0.0', 'data_size'='1.5E8', 'min_value'='1', 'max_value'='10');""" result_row = sql """show index stats mvTestDup mv1;""" assertEquals("mvTestDup", result_row[0][0]) assertEquals("mv1", result_row[0][1]) assertEquals("3443", result_row[0][2]) - assertTrue(Long.parseLong(result_row[0][3]) >= timestamp) - timestamp = System.currentTimeMillis(); sql """alter table mvTestDup index mv3 modify column `mva_MAX__``value2``` set stats ('row_count'='234234', 'ndv'='3.0', 'num_nulls'='0.0', 'data_size'='1.5E8', 'min_value'='1', 'max_value'='10');""" result_row = sql """show index stats mvTestDup mv3;""" assertEquals("mvTestDup", result_row[0][0]) assertEquals("mv3", result_row[0][1]) assertEquals("234234", result_row[0][2]) - assertTrue(Long.parseLong(result_row[0][3]) >= timestamp) sql """drop stats mvTestDup""" result_sample = sql """show column stats mvTestDup""" @@ -474,6 +474,35 @@ suite("test_analyze_mv") { logger.info(e.getMessage()); return; } + + if (!isCloudMode()) { + // Test row count report and report for nereids + result_row = sql """show index stats mvTestDup mvTestDup""" + assertEquals(1, result_row.size()) + assertEquals("mvTestDup", result_row[0][0]) + assertEquals("mvTestDup", result_row[0][1]) + assertEquals("6", result_row[0][3]) + assertEquals("6", result_row[0][4]) + result_row = sql """show index stats mvTestDup mv1""" + assertEquals(1, result_row.size()) + assertEquals("mvTestDup", result_row[0][0]) + assertEquals("mv1", result_row[0][1]) + assertEquals("6", result_row[0][3]) + assertEquals("6", result_row[0][4]) + result_row = sql """show index stats mvTestDup mv2""" + assertEquals(1, result_row.size()) + assertEquals("mvTestDup", result_row[0][0]) + assertEquals("mv2", result_row[0][1]) + assertEquals("6", result_row[0][3]) + assertEquals("6", result_row[0][4]) + result_row = sql """show index stats mvTestDup mv3""" + assertEquals(1, result_row.size()) + assertEquals("mvTestDup", result_row[0][0]) + assertEquals("mv3", result_row[0][1]) + assertEquals("4", result_row[0][3]) + assertEquals("4", result_row[0][4]) + } + sql """analyze table mvTestDup with sample rows 4000000""" wait_analyze_finish("mvTestDup") result_sample = sql """SHOW ANALYZE mvTestDup;""" @@ -621,6 +650,38 @@ suite("test_analyze_mv") { verifyTaskStatus(result_sample, "mva_MIN__`value3`", "mv3") verifyTaskStatus(result_sample, "mva_SUM__CAST(`value1` AS bigint)", "mv3") + if (!isCloudMode()) { + // Test row count report and report for nereids + sql """truncate table mvTestDup""" + result_row = sql """show index stats mvTestDup mv3""" + assertEquals(1, result_row.size()) + assertEquals("mvTestDup", result_row[0][0]) + assertEquals("mv3", result_row[0][1]) + assertEquals("0", result_row[0][3]) + assertEquals("-1", result_row[0][4]) + + for (int i = 0; i < 120; i++) { + result_row = sql """show index stats mvTestDup mv3""" + logger.info("mv3 stats: " + result_row) + if (result_row[0][4] == "0") { + break; + } + Thread.sleep(5000) + } + result_row = sql """show index stats mvTestDup mv3""" + assertEquals(1, result_row.size()) + assertEquals("mvTestDup", result_row[0][0]) + assertEquals("mv3", result_row[0][1]) + assertEquals("0", result_row[0][3]) + assertEquals("0", result_row[0][4]) + sql """insert into mvTestDup values (1, 2, 3, 4, 5), (1, 2, 3, 4, 5), (10, 20, 30, 40, 50), (10, 20, 30, 40, 50), (100, 200, 300, 400, 500), (1001, 2001, 3001, 4001, 5001);""" + result_row = sql """show index stats mvTestDup mv3""" + assertEquals(1, result_row.size()) + assertEquals("mvTestDup", result_row[0][0]) + assertEquals("mv3", result_row[0][1]) + assertEquals("-1", result_row[0][4]) + } + // Test alter column stats sql """drop stats mvTestDup""" sql """alter table mvTestDup modify column key1 set stats ('ndv'='1', 'num_nulls'='1', 'min_value'='10', 'max_value'='40', 'row_count'='50');"""