put version guessing behind a flag

duckdb · Nov 18, 2024 · 6ed670b · 6ed670b
1 parent 15d0cc3
commit 6ed670b
Show file tree

Hide file tree

Showing 7 changed files with 27 additions and 5 deletions.
diff --git a/src/common/iceberg.cpp b/src/common/iceberg.cpp
@@ -188,7 +188,7 @@ string GenerateMetaDataUrl(FileSystem &fs, const string &meta_path, string &tabl
 }
 
 
-string IcebergSnapshot::GetMetaDataPath(const string &path, FileSystem &fs, string metadata_compression_codec, string table_version = DEFAULT_TABLE_VERSION, string version_format = DEFAULT_TABLE_VERSION_FORMAT) {
+string IcebergSnapshot::GetMetaDataPath(ClientContext &context, const string &path, FileSystem &fs, string metadata_compression_codec, string table_version = DEFAULT_TABLE_VERSION, string version_format = DEFAULT_TABLE_VERSION_FORMAT) {
 	string version_hint;
 	string meta_path = fs.JoinPath(path, "metadata");
 	if (StringUtil::EndsWith(path, ".json")) {
@@ -210,6 +210,12 @@ string IcebergSnapshot::GetMetaDataPath(const string &path, FileSystem &fs, stri
 		version_hint = GetTableVersionFromHint(meta_path, fs, DEFAULT_VERSION_HINT_FILE);
 		return GenerateMetaDataUrl(fs, meta_path, version_hint, metadata_compression_codec, version_format);
 	} else {
+		Value result;
+		(void)context.TryGetCurrentSetting("unsafe_enable_version_guessing", result);
+		if (!result.GetValue<bool>()) {
+			throw InvalidInputException("No version was provided and no version-hint could be found, globbing the filesystem to locate the latest version is disabled by default as this is considered unsafe and could result in reading uncommitted data. To enable this use 'SET unsafe_enable_version_guessing = true;'");
+		}
+
 		// We need to guess from file paths
 		return GuessTableVersion(meta_path, fs, table_version, metadata_compression_codec, version_format);
 	}

diff --git a/src/iceberg_extension.cpp b/src/iceberg_extension.cpp
@@ -28,6 +28,14 @@ static void LoadInternal(DatabaseInstance &instance) {
 }
 
 void IcebergExtension::Load(DuckDB &db) {
+	auto &config = DBConfig::GetConfig(*db.instance);
+
+	config.AddExtensionOption(
+		"unsafe_enable_version_guessing",
+		"Enable globbing the filesystem (if possible) to find the latest version metadata. This could result in reading an uncommitted version.",
+		LogicalType::BOOLEAN,
+		Value::BOOLEAN(false)
+	);
 	LoadInternal(*db.instance);
 }
 std::string IcebergExtension::Name() {

diff --git a/src/iceberg_functions/iceberg_metadata.cpp b/src/iceberg_functions/iceberg_metadata.cpp
@@ -75,7 +75,7 @@ static unique_ptr<FunctionData> IcebergMetaDataBind(ClientContext &context, Tabl
 		}
 	}
 
-	auto iceberg_meta_path = IcebergSnapshot::GetMetaDataPath(iceberg_path, fs, metadata_compression_codec, table_version, version_name_format);
+	auto iceberg_meta_path = IcebergSnapshot::GetMetaDataPath(context, iceberg_path, fs, metadata_compression_codec, table_version, version_name_format);
 	IcebergSnapshot snapshot_to_scan;
 	if (input.inputs.size() > 1) {
 		if (input.inputs[1].type() == LogicalType::UBIGINT) {

diff --git a/src/iceberg_functions/iceberg_scan.cpp b/src/iceberg_functions/iceberg_scan.cpp
@@ -248,7 +248,7 @@ static unique_ptr<TableRef> IcebergScanBindReplace(ClientContext &context, Table
 			version_name_format = StringValue::Get(kv.second);
 		}
 	}
-	auto iceberg_meta_path = IcebergSnapshot::GetMetaDataPath(iceberg_path, fs, metadata_compression_codec, table_version, version_name_format);
+	auto iceberg_meta_path = IcebergSnapshot::GetMetaDataPath(context, iceberg_path, fs, metadata_compression_codec, table_version, version_name_format);
 	IcebergSnapshot snapshot_to_scan;
 	if (input.inputs.size() > 1) {
 		if (input.inputs[1].type() == LogicalType::UBIGINT) {

diff --git a/src/iceberg_functions/iceberg_snapshots.cpp b/src/iceberg_functions/iceberg_snapshots.cpp
@@ -33,7 +33,7 @@ struct IcebergSnapshotGlobalTableFunctionState : public GlobalTableFunctionState
 		FileSystem &fs = FileSystem::GetFileSystem(context);
 
 		auto iceberg_meta_path = IcebergSnapshot::GetMetaDataPath(
-		    bind_data.filename, fs, bind_data.metadata_compression_codec, bind_data.table_version, bind_data.version_name_format);
+		    context, bind_data.filename, fs, bind_data.metadata_compression_codec, bind_data.table_version, bind_data.version_name_format);
 		global_state->metadata_file = IcebergSnapshot::ReadMetaData(iceberg_meta_path, fs,  bind_data.metadata_compression_codec);
 		global_state->metadata_doc =
 		    yyjson_read(global_state->metadata_file.c_str(), global_state->metadata_file.size(), 0);

diff --git a/src/include/iceberg_metadata.hpp b/src/include/iceberg_metadata.hpp
@@ -77,7 +77,7 @@ class IcebergSnapshot {
 
 	static IcebergSnapshot ParseSnapShot(yyjson_val *snapshot, idx_t iceberg_format_version, idx_t schema_id,
 	                                     vector<yyjson_val *> &schemas, string metadata_compression_codec, bool skip_schema_inference);
-	static string GetMetaDataPath(const string &path, FileSystem &fs, string metadata_compression_codec, string table_version, string version_format);
+	static string GetMetaDataPath(ClientContext &context, const string &path, FileSystem &fs, string metadata_compression_codec, string table_version, string version_format);
 	static string ReadMetaData(const string &path, FileSystem &fs, string metadata_compression_codec);
 	static yyjson_val *GetSnapshots(const string &path, FileSystem &fs, string GetSnapshotByTimestamp);
 	static unique_ptr<SnapshotParseInfo> GetParseInfo(yyjson_doc &metadata_json);

diff --git a/test/sql/iceberg_metadata.test b/test/sql/iceberg_metadata.test
@@ -53,6 +53,14 @@ SELECT * FROM ICEBERG_METADATA('data/iceberg/lineitem_iceberg_nonexistent');
 ----
 IO Error: Cannot open "data/iceberg/lineitem_iceberg_nonexistent": Metadata directory does not exist
 
+statement error
+SELECT * FROM ICEBERG_METADATA('data/iceberg/lineitem_iceberg_no_hint', ALLOW_MOVED_PATHS=TRUE);
+----
+<REGEX>:.*SET unsafe_enable_version_guessing.*
+
+statement ok
+SET unsafe_enable_version_guessing = true;
+
 query IIIIIIII
 SELECT * FROM ICEBERG_METADATA('data/iceberg/lineitem_iceberg_no_hint', ALLOW_MOVED_PATHS=TRUE);
 ----