fix

apache · Dec 11, 2024 · 4c121d9 · 4c121d9
1 parent 201ca19
commit 4c121d9
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 11 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -33,6 +33,7 @@
 import org.apache.doris.datasource.SchemaCacheValue;
 import org.apache.doris.datasource.hudi.HudiSchemaCacheValue;
 import org.apache.doris.datasource.hudi.HudiUtils;
+import org.apache.doris.datasource.hudi.source.HudiLocalEngineContext;
 import org.apache.doris.datasource.iceberg.IcebergUtils;
 import org.apache.doris.datasource.mvcc.MvccSnapshot;
 import org.apache.doris.mtmv.MTMVBaseTableIf;
@@ -71,7 +72,10 @@
 import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hudi.common.config.HoodieMetadataConfig;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.view.FileSystemViewManager;
+import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -119,6 +123,8 @@ public class HMSExternalTable extends ExternalTable implements MTMVRelatedTableI
     private HoodieTableMetaClient hudiClient = null;
     private final byte[] hudiClientLock = new byte[0];
 
+    private volatile HoodieTableFileSystemView hudiFsView = null;
+    private final byte[] hudiFsViewLock = new byte[0];
 
     static {
         SUPPORTED_HIVE_FILE_FORMATS = Sets.newHashSet();
@@ -1008,4 +1014,21 @@ public HoodieTableMetaClient getHudiClient() {
             return hudiClient;
         }
     }
+
+    public HoodieTableFileSystemView getHudiFsView(HoodieTableMetaClient hudiClient) {
+        if (hudiFsView != null) {
+            return hudiFsView;
+        }
+        synchronized (hudiFsViewLock) {
+            if (hudiFsView != null) {
+                return hudiFsView;
+            }
+            // If we need to cache fsView later,
+            // we can use `FileSystemViewManager.createViewManagerWithTableMetadata` to get view manager
+            HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder().build();
+            HudiLocalEngineContext ctx = new HudiLocalEngineContext(hudiClient.getStorageConf());
+            hudiFsView = FileSystemViewManager.createInMemoryFileSystemView(ctx, hudiClient, metadataConfig);
+            return hudiFsView;
+        }
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
@@ -804,7 +804,7 @@ public static String showCreateTable(org.apache.hadoop.hive.metastore.api.Table
     }
 
     public static Schema getHudiTableSchema(HMSExternalTable table) {
-        HoodieTableMetaClient metaClient = getHudiClient(table);
+        HoodieTableMetaClient metaClient = table.getHudiClient();
         TableSchemaResolver schemaUtil = new TableSchemaResolver(metaClient);
         Schema hudiSchema;
         try {
@@ -828,10 +828,6 @@ public static <T> T ugiDoAs(Configuration conf, PrivilegedExceptionAction<T> act
         return HadoopUGI.ugiDoAs(krbConfig, action);
     }
 
-    public static HoodieTableMetaClient getHudiClient(HMSExternalTable table) {
-        return table.getHudiClient();
-    }
-
     public static Configuration getConfiguration(HMSExternalTable table) {
         Configuration conf = DFSFileSystem.getHdfsConf(table.getCatalog().ifNotSetFallbackToSimpleAuth());
         for (Map.Entry<String, String> entry : table.getHadoopProperties().entrySet()) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java
@@ -60,7 +60,6 @@
 import org.apache.hudi.common.table.HoodieTableMetaClient;
 import org.apache.hudi.common.table.timeline.HoodieInstant;
 import org.apache.hudi.common.table.timeline.HoodieTimeline;
-import org.apache.hudi.common.table.view.FileSystemViewManager;
 import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
 import org.apache.hudi.common.util.Option;
 import org.apache.hudi.metadata.FileSystemBackedTableMetadata;
@@ -170,7 +169,7 @@ protected void doInitialize() throws UserException {
         initBackendPolicy();
         initSchemaParams();
 
-        hudiClient = HiveMetaStoreClientHelper.getHudiClient(hmsTable);
+        hudiClient = hmsTable.getHudiClient();
         hudiClient.reloadActiveTimeline();
         basePath = hmsTable.getRemoteTable().getSd().getLocation();
         inputFormat = hmsTable.getRemoteTable().getSd().getInputFormat();
@@ -232,9 +231,7 @@ protected void doInitialize() throws UserException {
             LOG.info("Can't find metadataTable or metadataTable is not initialized for table:{}",
                     hmsTable.getName());
         }
-        // If we need to cache fsView later,
-        // we can use `FileSystemViewManager.createViewManagerWithTableMetadata` to get view manager
-        inMemoryFileSystemView = FileSystemViewManager.createInMemoryFileSystemView(ctx, hudiClient, metadataConfig);
+        inMemoryFileSystemView = hmsTable.getHudiFsView(hudiClient);
     }
 
     @Override

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java
@@ -187,7 +187,7 @@ public LogicalHudiScan withScanParams(HMSExternalTable table, TableScanParams sc
                     optParams.put(k, v);
                 }
             });
-            HoodieTableMetaClient hudiClient = HiveMetaStoreClientHelper.getHudiClient(table);
+            HoodieTableMetaClient hudiClient = table.getHudiClient();
             try {
                 boolean isCowOrRoTable = table.isHoodieCowTable();
                 if (isCowOrRoTable) {